2024-11-10 22:37:25,891 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-10 22:37:25,907 main DEBUG Took 0.013359 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-10 22:37:25,907 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-10 22:37:25,908 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-10 22:37:25,909 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-10 22:37:25,910 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 22:37:25,920 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-10 22:37:25,935 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 22:37:25,937 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 22:37:25,938 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 22:37:25,939 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 22:37:25,939 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 22:37:25,940 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 22:37:25,941 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 22:37:25,941 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 22:37:25,942 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 22:37:25,943 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 22:37:25,944 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 22:37:25,944 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 22:37:25,945 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 22:37:25,945 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 22:37:25,946 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 22:37:25,946 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 22:37:25,947 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 22:37:25,947 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 22:37:25,948 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 22:37:25,948 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 22:37:25,949 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 22:37:25,949 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 22:37:25,950 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 22:37:25,950 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 22:37:25,951 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 22:37:25,951 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-10 22:37:25,953 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 22:37:25,955 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-10 22:37:25,958 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-10 22:37:25,958 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-10 22:37:25,960 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-10 22:37:25,960 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-10 22:37:25,972 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-10 22:37:25,976 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-10 22:37:25,978 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-10 22:37:25,979 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-10 22:37:25,979 main DEBUG createAppenders(={Console}) 2024-11-10 22:37:25,980 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-11-10 22:37:25,980 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-10 22:37:25,981 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-11-10 22:37:25,982 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-10 22:37:25,982 main DEBUG OutputStream closed 2024-11-10 22:37:25,982 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-10 22:37:25,983 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-10 22:37:25,983 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-11-10 22:37:26,050 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-10 22:37:26,052 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-10 22:37:26,053 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-10 22:37:26,054 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-10 22:37:26,054 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-10 22:37:26,054 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-10 22:37:26,055 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-10 22:37:26,055 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-10 22:37:26,055 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-10 22:37:26,056 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-10 22:37:26,056 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-10 22:37:26,056 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-10 22:37:26,056 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-10 22:37:26,057 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-10 22:37:26,057 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-10 22:37:26,057 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-10 22:37:26,057 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-10 22:37:26,058 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-10 22:37:26,060 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-10 22:37:26,061 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-11-10 22:37:26,061 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-10 22:37:26,062 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-11-10T22:37:26,328 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3340d2ba-2e32-7278-81af-9687863d92cc 2024-11-10 22:37:26,331 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-10 22:37:26,331 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-10T22:37:26,340 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithNoInMemCompaction timeout: 13 mins 2024-11-10T22:37:26,359 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-10T22:37:26,373 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3340d2ba-2e32-7278-81af-9687863d92cc/cluster_f1d987d6-b0d7-db14-6c2b-f9cd24bc1de9, deleteOnExit=true 2024-11-10T22:37:26,374 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-10T22:37:26,375 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3340d2ba-2e32-7278-81af-9687863d92cc/test.cache.data in system properties and HBase conf 2024-11-10T22:37:26,375 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3340d2ba-2e32-7278-81af-9687863d92cc/hadoop.tmp.dir in system properties and HBase conf 2024-11-10T22:37:26,376 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3340d2ba-2e32-7278-81af-9687863d92cc/hadoop.log.dir in system properties and HBase conf 2024-11-10T22:37:26,376 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3340d2ba-2e32-7278-81af-9687863d92cc/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-10T22:37:26,377 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3340d2ba-2e32-7278-81af-9687863d92cc/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-10T22:37:26,377 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-10T22:37:26,466 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-10T22:37:26,551 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-10T22:37:26,555 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3340d2ba-2e32-7278-81af-9687863d92cc/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-10T22:37:26,556 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3340d2ba-2e32-7278-81af-9687863d92cc/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-10T22:37:26,556 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3340d2ba-2e32-7278-81af-9687863d92cc/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-10T22:37:26,556 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3340d2ba-2e32-7278-81af-9687863d92cc/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-10T22:37:26,557 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3340d2ba-2e32-7278-81af-9687863d92cc/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-10T22:37:26,557 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3340d2ba-2e32-7278-81af-9687863d92cc/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-10T22:37:26,558 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3340d2ba-2e32-7278-81af-9687863d92cc/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-10T22:37:26,558 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3340d2ba-2e32-7278-81af-9687863d92cc/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-10T22:37:26,559 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3340d2ba-2e32-7278-81af-9687863d92cc/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-10T22:37:26,559 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3340d2ba-2e32-7278-81af-9687863d92cc/nfs.dump.dir in system properties and HBase conf 2024-11-10T22:37:26,559 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3340d2ba-2e32-7278-81af-9687863d92cc/java.io.tmpdir in system properties and HBase conf 2024-11-10T22:37:26,560 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3340d2ba-2e32-7278-81af-9687863d92cc/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-10T22:37:26,560 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3340d2ba-2e32-7278-81af-9687863d92cc/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-10T22:37:26,560 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3340d2ba-2e32-7278-81af-9687863d92cc/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-10T22:37:27,365 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-10T22:37:27,449 INFO [Time-limited test {}] log.Log(170): Logging initialized @2229ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-10T22:37:27,530 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-10T22:37:27,592 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-10T22:37:27,614 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-10T22:37:27,614 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-10T22:37:27,616 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-10T22:37:27,631 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-10T22:37:27,633 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3340d2ba-2e32-7278-81af-9687863d92cc/hadoop.log.dir/,AVAILABLE} 2024-11-10T22:37:27,635 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-10T22:37:27,829 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4ded99d3{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3340d2ba-2e32-7278-81af-9687863d92cc/java.io.tmpdir/jetty-localhost-34347-hadoop-hdfs-3_4_1-tests_jar-_-any-17624231771766492532/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-10T22:37:27,842 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:34347} 2024-11-10T22:37:27,842 INFO [Time-limited test {}] server.Server(415): Started @2623ms 2024-11-10T22:37:28,238 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-10T22:37:28,245 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-10T22:37:28,246 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-10T22:37:28,246 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-10T22:37:28,246 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-10T22:37:28,247 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3340d2ba-2e32-7278-81af-9687863d92cc/hadoop.log.dir/,AVAILABLE} 2024-11-10T22:37:28,247 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-10T22:37:28,366 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4269bd31{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3340d2ba-2e32-7278-81af-9687863d92cc/java.io.tmpdir/jetty-localhost-39887-hadoop-hdfs-3_4_1-tests_jar-_-any-13231278687599051085/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-10T22:37:28,367 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:39887} 2024-11-10T22:37:28,367 INFO [Time-limited test {}] server.Server(415): Started @3148ms 2024-11-10T22:37:28,423 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-10T22:37:28,891 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3340d2ba-2e32-7278-81af-9687863d92cc/cluster_f1d987d6-b0d7-db14-6c2b-f9cd24bc1de9/data/data2/current/BP-1304540299-172.17.0.2-1731278247130/current, will proceed with Du for space computation calculation, 2024-11-10T22:37:28,891 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3340d2ba-2e32-7278-81af-9687863d92cc/cluster_f1d987d6-b0d7-db14-6c2b-f9cd24bc1de9/data/data1/current/BP-1304540299-172.17.0.2-1731278247130/current, will proceed with Du for space computation calculation, 2024-11-10T22:37:28,933 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-10T22:37:28,992 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x6a2fe39cdc7ef622 with lease ID 0xcdbf05a3a13a1271: Processing first storage report for DS-e9b608e2-44e7-4d94-a606-f1ca6b4271fa from datanode DatanodeRegistration(127.0.0.1:46149, datanodeUuid=96a73bbd-4590-4882-973d-216f9d2ea6ca, infoPort=39069, infoSecurePort=0, ipcPort=36649, storageInfo=lv=-57;cid=testClusterID;nsid=1426689941;c=1731278247130) 2024-11-10T22:37:28,996 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x6a2fe39cdc7ef622 with lease ID 0xcdbf05a3a13a1271: from storage DS-e9b608e2-44e7-4d94-a606-f1ca6b4271fa node DatanodeRegistration(127.0.0.1:46149, datanodeUuid=96a73bbd-4590-4882-973d-216f9d2ea6ca, infoPort=39069, infoSecurePort=0, ipcPort=36649, storageInfo=lv=-57;cid=testClusterID;nsid=1426689941;c=1731278247130), blocks: 0, hasStaleStorage: true, processing time: 3 msecs, invalidatedBlocks: 0 2024-11-10T22:37:28,996 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x6a2fe39cdc7ef622 with lease ID 0xcdbf05a3a13a1271: Processing first storage report for DS-44f2db9e-5ce1-4746-a541-56518194cdd3 from datanode DatanodeRegistration(127.0.0.1:46149, datanodeUuid=96a73bbd-4590-4882-973d-216f9d2ea6ca, infoPort=39069, infoSecurePort=0, ipcPort=36649, storageInfo=lv=-57;cid=testClusterID;nsid=1426689941;c=1731278247130) 2024-11-10T22:37:28,997 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x6a2fe39cdc7ef622 with lease ID 0xcdbf05a3a13a1271: from storage DS-44f2db9e-5ce1-4746-a541-56518194cdd3 node DatanodeRegistration(127.0.0.1:46149, datanodeUuid=96a73bbd-4590-4882-973d-216f9d2ea6ca, infoPort=39069, infoSecurePort=0, ipcPort=36649, storageInfo=lv=-57;cid=testClusterID;nsid=1426689941;c=1731278247130), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-10T22:37:29,067 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3340d2ba-2e32-7278-81af-9687863d92cc 2024-11-10T22:37:29,153 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3340d2ba-2e32-7278-81af-9687863d92cc/cluster_f1d987d6-b0d7-db14-6c2b-f9cd24bc1de9/zookeeper_0, clientPort=53159, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3340d2ba-2e32-7278-81af-9687863d92cc/cluster_f1d987d6-b0d7-db14-6c2b-f9cd24bc1de9/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3340d2ba-2e32-7278-81af-9687863d92cc/cluster_f1d987d6-b0d7-db14-6c2b-f9cd24bc1de9/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-10T22:37:29,163 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=53159 2024-11-10T22:37:29,175 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T22:37:29,179 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T22:37:29,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073741825_1001 (size=7) 2024-11-10T22:37:29,829 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39 with version=8 2024-11-10T22:37:29,829 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/hbase-staging 2024-11-10T22:37:29,926 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-10T22:37:30,175 INFO [Time-limited test {}] client.ConnectionUtils(128): master/849d0d71310f:0 server-side Connection retries=45 2024-11-10T22:37:30,186 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-10T22:37:30,187 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-10T22:37:30,191 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-10T22:37:30,191 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-10T22:37:30,192 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-10T22:37:30,333 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-10T22:37:30,396 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-10T22:37:30,405 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-10T22:37:30,409 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-10T22:37:30,435 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 12828 (auto-detected) 2024-11-10T22:37:30,436 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-10T22:37:30,455 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:43351 2024-11-10T22:37:30,477 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:43351 connecting to ZooKeeper ensemble=127.0.0.1:53159 2024-11-10T22:37:30,511 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:433510x0, quorum=127.0.0.1:53159, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-10T22:37:30,514 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:43351-0x101945512150000 connected 2024-11-10T22:37:30,543 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T22:37:30,546 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T22:37:30,555 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:43351-0x101945512150000, quorum=127.0.0.1:53159, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-10T22:37:30,559 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39, hbase.cluster.distributed=false 2024-11-10T22:37:30,582 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:43351-0x101945512150000, quorum=127.0.0.1:53159, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-10T22:37:30,586 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43351 2024-11-10T22:37:30,587 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43351 2024-11-10T22:37:30,588 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43351 2024-11-10T22:37:30,592 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43351 2024-11-10T22:37:30,592 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43351 2024-11-10T22:37:30,704 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/849d0d71310f:0 server-side Connection retries=45 2024-11-10T22:37:30,705 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-10T22:37:30,706 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-10T22:37:30,706 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-10T22:37:30,706 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-10T22:37:30,706 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-10T22:37:30,709 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-10T22:37:30,711 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-10T22:37:30,712 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:43753 2024-11-10T22:37:30,714 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:43753 connecting to ZooKeeper ensemble=127.0.0.1:53159 2024-11-10T22:37:30,715 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T22:37:30,718 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T22:37:30,726 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:437530x0, quorum=127.0.0.1:53159, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-10T22:37:30,726 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:43753-0x101945512150001 connected 2024-11-10T22:37:30,727 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43753-0x101945512150001, quorum=127.0.0.1:53159, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-10T22:37:30,731 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-10T22:37:30,740 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-10T22:37:30,742 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43753-0x101945512150001, quorum=127.0.0.1:53159, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-10T22:37:30,747 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43753-0x101945512150001, quorum=127.0.0.1:53159, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-10T22:37:30,748 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43753 2024-11-10T22:37:30,748 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43753 2024-11-10T22:37:30,749 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43753 2024-11-10T22:37:30,749 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43753 2024-11-10T22:37:30,750 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43753 2024-11-10T22:37:30,767 DEBUG [M:0;849d0d71310f:43351 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;849d0d71310f:43351 2024-11-10T22:37:30,768 INFO [master/849d0d71310f:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/849d0d71310f,43351,1731278249979 2024-11-10T22:37:30,774 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43753-0x101945512150001, quorum=127.0.0.1:53159, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-10T22:37:30,775 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43351-0x101945512150000, quorum=127.0.0.1:53159, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-10T22:37:30,776 DEBUG [master/849d0d71310f:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:43351-0x101945512150000, quorum=127.0.0.1:53159, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/849d0d71310f,43351,1731278249979 2024-11-10T22:37:30,796 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43753-0x101945512150001, quorum=127.0.0.1:53159, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-10T22:37:30,796 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43351-0x101945512150000, quorum=127.0.0.1:53159, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T22:37:30,796 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43753-0x101945512150001, quorum=127.0.0.1:53159, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T22:37:30,797 DEBUG [master/849d0d71310f:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:43351-0x101945512150000, quorum=127.0.0.1:53159, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-10T22:37:30,799 INFO [master/849d0d71310f:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/849d0d71310f,43351,1731278249979 from backup master directory 2024-11-10T22:37:30,802 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43753-0x101945512150001, quorum=127.0.0.1:53159, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-10T22:37:30,802 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43351-0x101945512150000, quorum=127.0.0.1:53159, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/849d0d71310f,43351,1731278249979 2024-11-10T22:37:30,803 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43351-0x101945512150000, quorum=127.0.0.1:53159, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-10T22:37:30,803 WARN [master/849d0d71310f:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-10T22:37:30,803 INFO [master/849d0d71310f:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=849d0d71310f,43351,1731278249979 2024-11-10T22:37:30,805 INFO [master/849d0d71310f:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-10T22:37:30,807 INFO [master/849d0d71310f:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-10T22:37:30,863 DEBUG [master/849d0d71310f:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/hbase.id] with ID: 59ccc382-681c-4202-a6b6-e337b1afc6c7 2024-11-10T22:37:30,864 DEBUG [master/849d0d71310f:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/.tmp/hbase.id 2024-11-10T22:37:30,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073741826_1002 (size=42) 2024-11-10T22:37:31,276 DEBUG [master/849d0d71310f:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/.tmp/hbase.id]:[hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/hbase.id] 2024-11-10T22:37:31,321 INFO [master/849d0d71310f:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T22:37:31,326 INFO [master/849d0d71310f:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-10T22:37:31,345 INFO [master/849d0d71310f:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 17ms. 2024-11-10T22:37:31,349 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43753-0x101945512150001, quorum=127.0.0.1:53159, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T22:37:31,349 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43351-0x101945512150000, quorum=127.0.0.1:53159, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T22:37:31,363 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073741827_1003 (size=196) 2024-11-10T22:37:31,781 INFO [master/849d0d71310f:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-10T22:37:31,783 INFO [master/849d0d71310f:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-10T22:37:31,800 DEBUG [master/849d0d71310f:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:37:31,805 INFO [master/849d0d71310f:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-10T22:37:31,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073741828_1004 (size=1189) 2024-11-10T22:37:32,254 INFO [master/849d0d71310f:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/MasterData/data/master/store 2024-11-10T22:37:32,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073741829_1005 (size=34) 2024-11-10T22:37:32,278 INFO [master/849d0d71310f:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-10T22:37:32,282 DEBUG [master/849d0d71310f:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T22:37:32,283 DEBUG [master/849d0d71310f:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-10T22:37:32,283 INFO [master/849d0d71310f:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T22:37:32,284 DEBUG [master/849d0d71310f:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T22:37:32,285 DEBUG [master/849d0d71310f:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-10T22:37:32,285 DEBUG [master/849d0d71310f:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T22:37:32,285 INFO [master/849d0d71310f:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T22:37:32,287 DEBUG [master/849d0d71310f:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731278252283Disabling compacts and flushes for region at 1731278252283Disabling writes for close at 1731278252285 (+2 ms)Writing region close event to WAL at 1731278252285Closed at 1731278252285 2024-11-10T22:37:32,289 WARN [master/849d0d71310f:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/MasterData/data/master/store/.initializing 2024-11-10T22:37:32,289 DEBUG [master/849d0d71310f:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/MasterData/WALs/849d0d71310f,43351,1731278249979 2024-11-10T22:37:32,298 INFO [master/849d0d71310f:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-10T22:37:32,313 INFO [master/849d0d71310f:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=849d0d71310f%2C43351%2C1731278249979, suffix=, logDir=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/MasterData/WALs/849d0d71310f,43351,1731278249979, archiveDir=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/MasterData/oldWALs, maxLogs=10 2024-11-10T22:37:32,335 DEBUG [master/849d0d71310f:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/MasterData/WALs/849d0d71310f,43351,1731278249979/849d0d71310f%2C43351%2C1731278249979.1731278252317, exclude list is [], retry=0 2024-11-10T22:37:32,352 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46149,DS-e9b608e2-44e7-4d94-a606-f1ca6b4271fa,DISK] 2024-11-10T22:37:32,355 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-10T22:37:32,392 INFO [master/849d0d71310f:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/MasterData/WALs/849d0d71310f,43351,1731278249979/849d0d71310f%2C43351%2C1731278249979.1731278252317 2024-11-10T22:37:32,393 DEBUG [master/849d0d71310f:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39069:39069)] 2024-11-10T22:37:32,393 DEBUG [master/849d0d71310f:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-10T22:37:32,394 DEBUG [master/849d0d71310f:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T22:37:32,397 DEBUG [master/849d0d71310f:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T22:37:32,398 DEBUG [master/849d0d71310f:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T22:37:32,435 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T22:37:32,463 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-10T22:37:32,467 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:37:32,470 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T22:37:32,470 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T22:37:32,474 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-10T22:37:32,474 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:37:32,475 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T22:37:32,476 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T22:37:32,478 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-10T22:37:32,479 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:37:32,480 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T22:37:32,480 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T22:37:32,482 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-10T22:37:32,483 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:37:32,484 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T22:37:32,484 DEBUG [master/849d0d71310f:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T22:37:32,487 DEBUG [master/849d0d71310f:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-10T22:37:32,488 DEBUG [master/849d0d71310f:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-10T22:37:32,493 DEBUG [master/849d0d71310f:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T22:37:32,494 DEBUG [master/849d0d71310f:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T22:37:32,498 DEBUG [master/849d0d71310f:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-10T22:37:32,501 DEBUG [master/849d0d71310f:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T22:37:32,506 DEBUG [master/849d0d71310f:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-10T22:37:32,507 INFO [master/849d0d71310f:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61125387, jitterRate=-0.08916075527667999}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-10T22:37:32,512 DEBUG [master/849d0d71310f:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731278252410Initializing all the Stores at 1731278252412 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731278252413 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731278252413Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731278252414 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731278252414Cleaning up temporary data from old regions at 1731278252494 (+80 ms)Region opened successfully at 1731278252512 (+18 ms) 2024-11-10T22:37:32,514 INFO [master/849d0d71310f:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-10T22:37:32,549 DEBUG [master/849d0d71310f:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a2d0089, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=849d0d71310f/172.17.0.2:0 2024-11-10T22:37:32,583 INFO [master/849d0d71310f:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-10T22:37:32,596 INFO [master/849d0d71310f:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-10T22:37:32,596 INFO [master/849d0d71310f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-10T22:37:32,600 INFO [master/849d0d71310f:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-10T22:37:32,601 INFO [master/849d0d71310f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-10T22:37:32,607 INFO [master/849d0d71310f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-10T22:37:32,607 INFO [master/849d0d71310f:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-10T22:37:32,636 INFO [master/849d0d71310f:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-10T22:37:32,647 DEBUG [master/849d0d71310f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43351-0x101945512150000, quorum=127.0.0.1:53159, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-10T22:37:32,650 DEBUG [master/849d0d71310f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-10T22:37:32,654 INFO [master/849d0d71310f:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-10T22:37:32,655 DEBUG [master/849d0d71310f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43351-0x101945512150000, quorum=127.0.0.1:53159, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-10T22:37:32,658 DEBUG [master/849d0d71310f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-10T22:37:32,660 INFO [master/849d0d71310f:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-10T22:37:32,664 DEBUG [master/849d0d71310f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43351-0x101945512150000, quorum=127.0.0.1:53159, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-10T22:37:32,666 DEBUG [master/849d0d71310f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-10T22:37:32,667 DEBUG [master/849d0d71310f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43351-0x101945512150000, quorum=127.0.0.1:53159, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-10T22:37:32,669 DEBUG [master/849d0d71310f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-10T22:37:32,689 DEBUG [master/849d0d71310f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43351-0x101945512150000, quorum=127.0.0.1:53159, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-10T22:37:32,690 DEBUG [master/849d0d71310f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-10T22:37:32,695 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43753-0x101945512150001, quorum=127.0.0.1:53159, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-10T22:37:32,695 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43351-0x101945512150000, quorum=127.0.0.1:53159, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-10T22:37:32,695 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43753-0x101945512150001, quorum=127.0.0.1:53159, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T22:37:32,696 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43351-0x101945512150000, quorum=127.0.0.1:53159, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T22:37:32,698 INFO [master/849d0d71310f:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=849d0d71310f,43351,1731278249979, sessionid=0x101945512150000, setting cluster-up flag (Was=false) 2024-11-10T22:37:32,711 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43351-0x101945512150000, quorum=127.0.0.1:53159, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T22:37:32,711 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43753-0x101945512150001, quorum=127.0.0.1:53159, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T22:37:32,717 DEBUG [master/849d0d71310f:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-10T22:37:32,719 DEBUG [master/849d0d71310f:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=849d0d71310f,43351,1731278249979 2024-11-10T22:37:32,726 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43351-0x101945512150000, quorum=127.0.0.1:53159, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T22:37:32,726 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43753-0x101945512150001, quorum=127.0.0.1:53159, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T22:37:32,732 DEBUG [master/849d0d71310f:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-10T22:37:32,734 DEBUG [master/849d0d71310f:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=849d0d71310f,43351,1731278249979 2024-11-10T22:37:32,740 INFO [master/849d0d71310f:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-10T22:37:32,756 INFO [RS:0;849d0d71310f:43753 {}] regionserver.HRegionServer(746): ClusterId : 59ccc382-681c-4202-a6b6-e337b1afc6c7 2024-11-10T22:37:32,760 DEBUG [RS:0;849d0d71310f:43753 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-10T22:37:32,765 DEBUG [RS:0;849d0d71310f:43753 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-10T22:37:32,765 DEBUG [RS:0;849d0d71310f:43753 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-10T22:37:32,769 DEBUG [RS:0;849d0d71310f:43753 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-10T22:37:32,769 DEBUG [RS:0;849d0d71310f:43753 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36e76523, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=849d0d71310f/172.17.0.2:0 2024-11-10T22:37:32,788 DEBUG [RS:0;849d0d71310f:43753 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;849d0d71310f:43753 2024-11-10T22:37:32,793 INFO [RS:0;849d0d71310f:43753 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-10T22:37:32,793 INFO [RS:0;849d0d71310f:43753 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-10T22:37:32,793 DEBUG [RS:0;849d0d71310f:43753 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-10T22:37:32,797 INFO [RS:0;849d0d71310f:43753 {}] regionserver.HRegionServer(2659): reportForDuty to master=849d0d71310f,43351,1731278249979 with port=43753, startcode=1731278250663 2024-11-10T22:37:32,811 DEBUG [RS:0;849d0d71310f:43753 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-10T22:37:32,817 DEBUG [master/849d0d71310f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-10T22:37:32,827 INFO [master/849d0d71310f:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-10T22:37:32,834 INFO [master/849d0d71310f:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-10T22:37:32,839 DEBUG [master/849d0d71310f:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 849d0d71310f,43351,1731278249979 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-10T22:37:32,846 DEBUG [master/849d0d71310f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/849d0d71310f:0, corePoolSize=5, maxPoolSize=5 2024-11-10T22:37:32,846 DEBUG [master/849d0d71310f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/849d0d71310f:0, corePoolSize=5, maxPoolSize=5 2024-11-10T22:37:32,847 DEBUG [master/849d0d71310f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/849d0d71310f:0, corePoolSize=5, maxPoolSize=5 2024-11-10T22:37:32,847 DEBUG [master/849d0d71310f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/849d0d71310f:0, corePoolSize=5, maxPoolSize=5 2024-11-10T22:37:32,847 DEBUG [master/849d0d71310f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/849d0d71310f:0, corePoolSize=10, maxPoolSize=10 2024-11-10T22:37:32,847 DEBUG [master/849d0d71310f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/849d0d71310f:0, corePoolSize=1, maxPoolSize=1 2024-11-10T22:37:32,847 DEBUG [master/849d0d71310f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/849d0d71310f:0, corePoolSize=2, maxPoolSize=2 2024-11-10T22:37:32,848 DEBUG [master/849d0d71310f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/849d0d71310f:0, corePoolSize=1, maxPoolSize=1 2024-11-10T22:37:32,849 INFO [master/849d0d71310f:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731278282849 2024-11-10T22:37:32,852 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49277, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-10T22:37:32,852 INFO [master/849d0d71310f:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-10T22:37:32,853 INFO [master/849d0d71310f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-10T22:37:32,855 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-10T22:37:32,856 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-10T22:37:32,857 INFO [master/849d0d71310f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-10T22:37:32,857 INFO [master/849d0d71310f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-10T22:37:32,858 INFO [master/849d0d71310f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-10T22:37:32,858 INFO [master/849d0d71310f:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-10T22:37:32,860 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43351 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:32,862 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:37:32,860 INFO [master/849d0d71310f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-10T22:37:32,862 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-10T22:37:32,864 INFO [master/849d0d71310f:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-10T22:37:32,865 INFO [master/849d0d71310f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-10T22:37:32,866 INFO [master/849d0d71310f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-10T22:37:32,868 INFO [master/849d0d71310f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-10T22:37:32,869 INFO [master/849d0d71310f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-10T22:37:32,872 DEBUG [master/849d0d71310f:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/849d0d71310f:0:becomeActiveMaster-HFileCleaner.large.0-1731278252870,5,FailOnTimeoutGroup] 2024-11-10T22:37:32,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073741831_1007 (size=1321) 2024-11-10T22:37:32,876 DEBUG [master/849d0d71310f:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/849d0d71310f:0:becomeActiveMaster-HFileCleaner.small.0-1731278252872,5,FailOnTimeoutGroup] 2024-11-10T22:37:32,876 INFO [master/849d0d71310f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-10T22:37:32,876 INFO [master/849d0d71310f:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-10T22:37:32,878 INFO [master/849d0d71310f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-10T22:37:32,878 INFO [master/849d0d71310f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-10T22:37:32,878 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-10T22:37:32,879 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39 2024-11-10T22:37:32,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073741832_1008 (size=32) 2024-11-10T22:37:32,892 DEBUG [RS:0;849d0d71310f:43753 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-10T22:37:32,892 WARN [RS:0;849d0d71310f:43753 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-10T22:37:32,994 INFO [RS:0;849d0d71310f:43753 {}] regionserver.HRegionServer(2659): reportForDuty to master=849d0d71310f,43351,1731278249979 with port=43753, startcode=1731278250663 2024-11-10T22:37:32,996 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43351 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 849d0d71310f,43753,1731278250663 2024-11-10T22:37:32,999 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43351 {}] master.ServerManager(517): Registering regionserver=849d0d71310f,43753,1731278250663 2024-11-10T22:37:33,007 DEBUG [RS:0;849d0d71310f:43753 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39 2024-11-10T22:37:33,007 DEBUG [RS:0;849d0d71310f:43753 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:44923 2024-11-10T22:37:33,007 DEBUG [RS:0;849d0d71310f:43753 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-10T22:37:33,012 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43351-0x101945512150000, quorum=127.0.0.1:53159, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-10T22:37:33,012 DEBUG [RS:0;849d0d71310f:43753 {}] zookeeper.ZKUtil(111): regionserver:43753-0x101945512150001, quorum=127.0.0.1:53159, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/849d0d71310f,43753,1731278250663 2024-11-10T22:37:33,013 WARN [RS:0;849d0d71310f:43753 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-10T22:37:33,013 INFO [RS:0;849d0d71310f:43753 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-10T22:37:33,013 DEBUG [RS:0;849d0d71310f:43753 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/WALs/849d0d71310f,43753,1731278250663 2024-11-10T22:37:33,015 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [849d0d71310f,43753,1731278250663] 2024-11-10T22:37:33,039 INFO [RS:0;849d0d71310f:43753 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-10T22:37:33,052 INFO [RS:0;849d0d71310f:43753 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-10T22:37:33,057 INFO [RS:0;849d0d71310f:43753 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-10T22:37:33,057 INFO [RS:0;849d0d71310f:43753 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T22:37:33,058 INFO [RS:0;849d0d71310f:43753 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-10T22:37:33,063 INFO [RS:0;849d0d71310f:43753 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-10T22:37:33,065 INFO [RS:0;849d0d71310f:43753 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-10T22:37:33,065 DEBUG [RS:0;849d0d71310f:43753 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/849d0d71310f:0, corePoolSize=1, maxPoolSize=1 2024-11-10T22:37:33,066 DEBUG [RS:0;849d0d71310f:43753 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/849d0d71310f:0, corePoolSize=1, maxPoolSize=1 2024-11-10T22:37:33,066 DEBUG [RS:0;849d0d71310f:43753 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/849d0d71310f:0, corePoolSize=1, maxPoolSize=1 2024-11-10T22:37:33,066 DEBUG [RS:0;849d0d71310f:43753 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/849d0d71310f:0, corePoolSize=1, maxPoolSize=1 2024-11-10T22:37:33,066 DEBUG [RS:0;849d0d71310f:43753 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/849d0d71310f:0, corePoolSize=1, maxPoolSize=1 2024-11-10T22:37:33,066 DEBUG [RS:0;849d0d71310f:43753 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/849d0d71310f:0, corePoolSize=2, maxPoolSize=2 2024-11-10T22:37:33,067 DEBUG [RS:0;849d0d71310f:43753 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/849d0d71310f:0, corePoolSize=1, maxPoolSize=1 2024-11-10T22:37:33,067 DEBUG [RS:0;849d0d71310f:43753 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/849d0d71310f:0, corePoolSize=1, maxPoolSize=1 2024-11-10T22:37:33,067 DEBUG [RS:0;849d0d71310f:43753 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/849d0d71310f:0, corePoolSize=1, maxPoolSize=1 2024-11-10T22:37:33,067 DEBUG [RS:0;849d0d71310f:43753 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/849d0d71310f:0, corePoolSize=1, maxPoolSize=1 2024-11-10T22:37:33,068 DEBUG [RS:0;849d0d71310f:43753 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/849d0d71310f:0, corePoolSize=1, maxPoolSize=1 2024-11-10T22:37:33,068 DEBUG [RS:0;849d0d71310f:43753 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/849d0d71310f:0, corePoolSize=1, maxPoolSize=1 2024-11-10T22:37:33,068 DEBUG [RS:0;849d0d71310f:43753 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/849d0d71310f:0, corePoolSize=3, maxPoolSize=3 2024-11-10T22:37:33,068 DEBUG [RS:0;849d0d71310f:43753 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0, corePoolSize=3, maxPoolSize=3 2024-11-10T22:37:33,069 INFO [RS:0;849d0d71310f:43753 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-10T22:37:33,069 INFO [RS:0;849d0d71310f:43753 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-10T22:37:33,069 INFO [RS:0;849d0d71310f:43753 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T22:37:33,070 INFO [RS:0;849d0d71310f:43753 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-10T22:37:33,070 INFO [RS:0;849d0d71310f:43753 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-10T22:37:33,070 INFO [RS:0;849d0d71310f:43753 {}] hbase.ChoreService(168): Chore ScheduledChore name=849d0d71310f,43753,1731278250663-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-10T22:37:33,097 INFO [RS:0;849d0d71310f:43753 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-10T22:37:33,099 INFO [RS:0;849d0d71310f:43753 {}] hbase.ChoreService(168): Chore ScheduledChore name=849d0d71310f,43753,1731278250663-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T22:37:33,099 INFO [RS:0;849d0d71310f:43753 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T22:37:33,099 INFO [RS:0;849d0d71310f:43753 {}] regionserver.Replication(171): 849d0d71310f,43753,1731278250663 started 2024-11-10T22:37:33,119 INFO [RS:0;849d0d71310f:43753 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T22:37:33,120 INFO [RS:0;849d0d71310f:43753 {}] regionserver.HRegionServer(1482): Serving as 849d0d71310f,43753,1731278250663, RpcServer on 849d0d71310f/172.17.0.2:43753, sessionid=0x101945512150001 2024-11-10T22:37:33,121 DEBUG [RS:0;849d0d71310f:43753 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-10T22:37:33,121 DEBUG [RS:0;849d0d71310f:43753 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 849d0d71310f,43753,1731278250663 2024-11-10T22:37:33,122 DEBUG [RS:0;849d0d71310f:43753 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '849d0d71310f,43753,1731278250663' 2024-11-10T22:37:33,122 DEBUG [RS:0;849d0d71310f:43753 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-10T22:37:33,123 DEBUG [RS:0;849d0d71310f:43753 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-10T22:37:33,124 DEBUG [RS:0;849d0d71310f:43753 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-10T22:37:33,124 DEBUG [RS:0;849d0d71310f:43753 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-10T22:37:33,124 DEBUG [RS:0;849d0d71310f:43753 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 849d0d71310f,43753,1731278250663 2024-11-10T22:37:33,124 DEBUG [RS:0;849d0d71310f:43753 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '849d0d71310f,43753,1731278250663' 2024-11-10T22:37:33,124 DEBUG [RS:0;849d0d71310f:43753 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-10T22:37:33,125 DEBUG [RS:0;849d0d71310f:43753 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-10T22:37:33,126 DEBUG [RS:0;849d0d71310f:43753 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-10T22:37:33,126 INFO [RS:0;849d0d71310f:43753 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-10T22:37:33,126 INFO [RS:0;849d0d71310f:43753 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-10T22:37:33,232 INFO [RS:0;849d0d71310f:43753 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-10T22:37:33,235 INFO [RS:0;849d0d71310f:43753 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=849d0d71310f%2C43753%2C1731278250663, suffix=, logDir=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/WALs/849d0d71310f,43753,1731278250663, archiveDir=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/oldWALs, maxLogs=32 2024-11-10T22:37:33,253 DEBUG [RS:0;849d0d71310f:43753 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/WALs/849d0d71310f,43753,1731278250663/849d0d71310f%2C43753%2C1731278250663.1731278253238, exclude list is [], retry=0 2024-11-10T22:37:33,258 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46149,DS-e9b608e2-44e7-4d94-a606-f1ca6b4271fa,DISK] 2024-11-10T22:37:33,262 INFO [RS:0;849d0d71310f:43753 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/WALs/849d0d71310f,43753,1731278250663/849d0d71310f%2C43753%2C1731278250663.1731278253238 2024-11-10T22:37:33,263 DEBUG [RS:0;849d0d71310f:43753 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39069:39069)] 2024-11-10T22:37:33,290 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T22:37:33,293 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-10T22:37:33,296 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-10T22:37:33,296 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:37:33,297 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T22:37:33,297 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-10T22:37:33,299 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-10T22:37:33,299 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:37:33,300 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T22:37:33,301 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-10T22:37:33,303 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-10T22:37:33,303 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:37:33,304 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T22:37:33,304 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-10T22:37:33,306 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-10T22:37:33,306 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:37:33,307 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T22:37:33,307 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-10T22:37:33,309 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/hbase/meta/1588230740 2024-11-10T22:37:33,309 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/hbase/meta/1588230740 2024-11-10T22:37:33,312 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-10T22:37:33,312 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-10T22:37:33,313 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-10T22:37:33,315 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-10T22:37:33,319 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-10T22:37:33,320 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66994799, jitterRate=-0.001699700951576233}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-10T22:37:33,322 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731278253291Initializing all the Stores at 1731278253292 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731278253293 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731278253293Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731278253293Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731278253293Cleaning up temporary data from old regions at 1731278253312 (+19 ms)Region opened successfully at 1731278253322 (+10 ms) 2024-11-10T22:37:33,322 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-10T22:37:33,322 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-10T22:37:33,323 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-10T22:37:33,323 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-10T22:37:33,323 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-10T22:37:33,324 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-10T22:37:33,324 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731278253322Disabling compacts and flushes for region at 1731278253322Disabling writes for close at 1731278253323 (+1 ms)Writing region close event to WAL at 1731278253324 (+1 ms)Closed at 1731278253324 2024-11-10T22:37:33,327 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-10T22:37:33,327 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-10T22:37:33,334 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-10T22:37:33,342 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-10T22:37:33,345 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-10T22:37:33,498 DEBUG [849d0d71310f:43351 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-10T22:37:33,510 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=849d0d71310f,43753,1731278250663 2024-11-10T22:37:33,516 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 849d0d71310f,43753,1731278250663, state=OPENING 2024-11-10T22:37:33,521 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-10T22:37:33,523 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43753-0x101945512150001, quorum=127.0.0.1:53159, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T22:37:33,523 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43351-0x101945512150000, quorum=127.0.0.1:53159, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T22:37:33,524 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-10T22:37:33,524 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-10T22:37:33,525 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-10T22:37:33,527 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=849d0d71310f,43753,1731278250663}] 2024-11-10T22:37:33,703 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-10T22:37:33,707 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39591, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-10T22:37:33,718 INFO [RS_OPEN_META-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-10T22:37:33,718 INFO [RS_OPEN_META-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-10T22:37:33,719 INFO [RS_OPEN_META-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-10T22:37:33,722 INFO [RS_OPEN_META-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=849d0d71310f%2C43753%2C1731278250663.meta, suffix=.meta, logDir=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/WALs/849d0d71310f,43753,1731278250663, archiveDir=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/oldWALs, maxLogs=32 2024-11-10T22:37:33,740 DEBUG [RS_OPEN_META-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/WALs/849d0d71310f,43753,1731278250663/849d0d71310f%2C43753%2C1731278250663.meta.1731278253724.meta, exclude list is [], retry=0 2024-11-10T22:37:33,744 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46149,DS-e9b608e2-44e7-4d94-a606-f1ca6b4271fa,DISK] 2024-11-10T22:37:33,747 INFO [RS_OPEN_META-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/WALs/849d0d71310f,43753,1731278250663/849d0d71310f%2C43753%2C1731278250663.meta.1731278253724.meta 2024-11-10T22:37:33,748 DEBUG [RS_OPEN_META-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39069:39069)] 2024-11-10T22:37:33,748 DEBUG [RS_OPEN_META-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-10T22:37:33,751 DEBUG [RS_OPEN_META-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-10T22:37:33,754 DEBUG [RS_OPEN_META-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-10T22:37:33,759 INFO [RS_OPEN_META-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-10T22:37:33,764 DEBUG [RS_OPEN_META-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-10T22:37:33,764 DEBUG [RS_OPEN_META-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T22:37:33,764 DEBUG [RS_OPEN_META-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-10T22:37:33,764 DEBUG [RS_OPEN_META-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-10T22:37:33,768 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-10T22:37:33,769 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-10T22:37:33,769 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:37:33,770 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T22:37:33,770 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-10T22:37:33,772 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-10T22:37:33,772 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:37:33,773 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T22:37:33,773 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-10T22:37:33,774 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-10T22:37:33,774 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:37:33,775 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T22:37:33,775 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-10T22:37:33,776 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-10T22:37:33,777 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:37:33,777 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T22:37:33,778 DEBUG [RS_OPEN_META-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-10T22:37:33,779 DEBUG [RS_OPEN_META-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/hbase/meta/1588230740 2024-11-10T22:37:33,781 DEBUG [RS_OPEN_META-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/hbase/meta/1588230740 2024-11-10T22:37:33,784 DEBUG [RS_OPEN_META-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-10T22:37:33,784 DEBUG [RS_OPEN_META-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-10T22:37:33,785 DEBUG [RS_OPEN_META-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-10T22:37:33,787 DEBUG [RS_OPEN_META-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-10T22:37:33,789 INFO [RS_OPEN_META-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61456063, jitterRate=-0.08423329889774323}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-10T22:37:33,789 DEBUG [RS_OPEN_META-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-10T22:37:33,791 DEBUG [RS_OPEN_META-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731278253765Writing region info on filesystem at 1731278253765Initializing all the Stores at 1731278253767 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731278253767Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731278253767Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731278253768 (+1 ms)Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731278253768Cleaning up temporary data from old regions at 1731278253784 (+16 ms)Running coprocessor post-open hooks at 1731278253790 (+6 ms)Region opened successfully at 1731278253791 (+1 ms) 2024-11-10T22:37:33,799 INFO [RS_OPEN_META-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731278253694 2024-11-10T22:37:33,812 DEBUG [RS_OPEN_META-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-10T22:37:33,813 INFO [RS_OPEN_META-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-10T22:37:33,814 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=849d0d71310f,43753,1731278250663 2024-11-10T22:37:33,818 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 849d0d71310f,43753,1731278250663, state=OPEN 2024-11-10T22:37:33,823 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43753-0x101945512150001, quorum=127.0.0.1:53159, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-10T22:37:33,823 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43351-0x101945512150000, quorum=127.0.0.1:53159, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-10T22:37:33,824 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-10T22:37:33,824 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-10T22:37:33,824 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:33,829 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-10T22:37:33,830 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=849d0d71310f,43753,1731278250663 in 297 msec 2024-11-10T22:37:33,836 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-10T22:37:33,837 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 497 msec 2024-11-10T22:37:33,838 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-10T22:37:33,839 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-10T22:37:33,860 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T22:37:33,861 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849d0d71310f,43753,1731278250663, seqNum=-1] 2024-11-10T22:37:33,883 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T22:37:33,885 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43085, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T22:37:33,908 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.1340 sec 2024-11-10T22:37:33,908 INFO [master/849d0d71310f:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731278253908, completionTime=-1 2024-11-10T22:37:33,912 INFO [master/849d0d71310f:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-10T22:37:33,912 DEBUG [master/849d0d71310f:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-10T22:37:33,940 INFO [master/849d0d71310f:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-10T22:37:33,940 INFO [master/849d0d71310f:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731278313940 2024-11-10T22:37:33,940 INFO [master/849d0d71310f:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731278373940 2024-11-10T22:37:33,940 INFO [master/849d0d71310f:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 28 msec 2024-11-10T22:37:33,942 INFO [master/849d0d71310f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=849d0d71310f,43351,1731278249979-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T22:37:33,943 INFO [master/849d0d71310f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=849d0d71310f,43351,1731278249979-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T22:37:33,943 INFO [master/849d0d71310f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=849d0d71310f,43351,1731278249979-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T22:37:33,944 INFO [master/849d0d71310f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-849d0d71310f:43351, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T22:37:33,945 INFO [master/849d0d71310f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-10T22:37:33,945 INFO [master/849d0d71310f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-10T22:37:33,951 DEBUG [master/849d0d71310f:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-10T22:37:33,974 INFO [master/849d0d71310f:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 3.171sec 2024-11-10T22:37:33,975 INFO [master/849d0d71310f:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-10T22:37:33,977 INFO [master/849d0d71310f:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-10T22:37:33,978 INFO [master/849d0d71310f:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-10T22:37:33,979 INFO [master/849d0d71310f:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-10T22:37:33,979 INFO [master/849d0d71310f:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-10T22:37:33,980 INFO [master/849d0d71310f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=849d0d71310f,43351,1731278249979-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-10T22:37:33,981 INFO [master/849d0d71310f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=849d0d71310f,43351,1731278249979-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-10T22:37:33,989 DEBUG [master/849d0d71310f:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-10T22:37:33,990 INFO [master/849d0d71310f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-10T22:37:33,990 INFO [master/849d0d71310f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=849d0d71310f,43351,1731278249979-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T22:37:34,068 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@343aff6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:37:34,070 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-10T22:37:34,070 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-10T22:37:34,073 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849d0d71310f,43351,-1 for getting cluster id 2024-11-10T22:37:34,076 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T22:37:34,084 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59ccc382-681c-4202-a6b6-e337b1afc6c7' 2024-11-10T22:37:34,087 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T22:37:34,087 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59ccc382-681c-4202-a6b6-e337b1afc6c7" 2024-11-10T22:37:34,090 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f7a823f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:37:34,090 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849d0d71310f,43351,-1] 2024-11-10T22:37:34,092 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T22:37:34,094 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:37:34,095 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46928, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T22:37:34,098 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32372268, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:37:34,099 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T22:37:34,106 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849d0d71310f,43753,1731278250663, seqNum=-1] 2024-11-10T22:37:34,106 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T22:37:34,109 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43628, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T22:37:34,130 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=849d0d71310f,43351,1731278249979 2024-11-10T22:37:34,143 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMixedAtomicity Thread=211, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=39, ProcessCount=11, AvailableMemoryMB=7735 2024-11-10T22:37:34,152 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-10T22:37:34,158 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 849d0d71310f,43351,1731278249979 2024-11-10T22:37:34,178 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@1109f50e 2024-11-10T22:37:34,179 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-10T22:37:34,182 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46944, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-10T22:37:34,184 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43351 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-10T22:37:34,188 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43351 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-10T22:37:34,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43351 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-10T22:37:34,202 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-10T22:37:34,204 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43351 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-11-10T22:37:34,205 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:37:34,207 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-10T22:37:34,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-10T22:37:34,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073741835_1011 (size=959) 2024-11-10T22:37:34,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-10T22:37:34,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-10T22:37:34,628 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => dbbcde27e8d425fcc5932f5841a8c339, NAME => 'TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39 2024-11-10T22:37:34,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073741836_1012 (size=53) 2024-11-10T22:37:34,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-10T22:37:35,038 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T22:37:35,038 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing dbbcde27e8d425fcc5932f5841a8c339, disabling compactions & flushes 2024-11-10T22:37:35,038 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. 2024-11-10T22:37:35,038 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. 2024-11-10T22:37:35,038 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. after waiting 0 ms 2024-11-10T22:37:35,038 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. 2024-11-10T22:37:35,039 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. 2024-11-10T22:37:35,039 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for dbbcde27e8d425fcc5932f5841a8c339: Waiting for close lock at 1731278255038Disabling compacts and flushes for region at 1731278255038Disabling writes for close at 1731278255038Writing region close event to WAL at 1731278255038Closed at 1731278255038 2024-11-10T22:37:35,041 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-10T22:37:35,047 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731278255041"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731278255041"}]},"ts":"1731278255041"} 2024-11-10T22:37:35,051 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-10T22:37:35,053 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-10T22:37:35,056 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731278255053"}]},"ts":"1731278255053"} 2024-11-10T22:37:35,060 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-10T22:37:35,062 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=dbbcde27e8d425fcc5932f5841a8c339, ASSIGN}] 2024-11-10T22:37:35,065 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=dbbcde27e8d425fcc5932f5841a8c339, ASSIGN 2024-11-10T22:37:35,066 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=dbbcde27e8d425fcc5932f5841a8c339, ASSIGN; state=OFFLINE, location=849d0d71310f,43753,1731278250663; forceNewPlan=false, retain=false 2024-11-10T22:37:35,218 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=dbbcde27e8d425fcc5932f5841a8c339, regionState=OPENING, regionLocation=849d0d71310f,43753,1731278250663 2024-11-10T22:37:35,222 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=dbbcde27e8d425fcc5932f5841a8c339, ASSIGN because future has completed 2024-11-10T22:37:35,223 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663}] 2024-11-10T22:37:35,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-10T22:37:35,385 INFO [RS_OPEN_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. 2024-11-10T22:37:35,385 DEBUG [RS_OPEN_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => dbbcde27e8d425fcc5932f5841a8c339, NAME => 'TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.', STARTKEY => '', ENDKEY => ''} 2024-11-10T22:37:35,386 DEBUG [RS_OPEN_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees dbbcde27e8d425fcc5932f5841a8c339 2024-11-10T22:37:35,386 DEBUG [RS_OPEN_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T22:37:35,386 DEBUG [RS_OPEN_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for dbbcde27e8d425fcc5932f5841a8c339 2024-11-10T22:37:35,386 DEBUG [RS_OPEN_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for dbbcde27e8d425fcc5932f5841a8c339 2024-11-10T22:37:35,388 INFO [StoreOpener-dbbcde27e8d425fcc5932f5841a8c339-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region dbbcde27e8d425fcc5932f5841a8c339 2024-11-10T22:37:35,391 INFO [StoreOpener-dbbcde27e8d425fcc5932f5841a8c339-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dbbcde27e8d425fcc5932f5841a8c339 columnFamilyName A 2024-11-10T22:37:35,391 DEBUG [StoreOpener-dbbcde27e8d425fcc5932f5841a8c339-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:37:35,392 INFO [StoreOpener-dbbcde27e8d425fcc5932f5841a8c339-1 {}] regionserver.HStore(327): Store=dbbcde27e8d425fcc5932f5841a8c339/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T22:37:35,392 INFO [StoreOpener-dbbcde27e8d425fcc5932f5841a8c339-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region dbbcde27e8d425fcc5932f5841a8c339 2024-11-10T22:37:35,394 INFO [StoreOpener-dbbcde27e8d425fcc5932f5841a8c339-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dbbcde27e8d425fcc5932f5841a8c339 columnFamilyName B 2024-11-10T22:37:35,394 DEBUG [StoreOpener-dbbcde27e8d425fcc5932f5841a8c339-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:37:35,395 INFO [StoreOpener-dbbcde27e8d425fcc5932f5841a8c339-1 {}] regionserver.HStore(327): Store=dbbcde27e8d425fcc5932f5841a8c339/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T22:37:35,395 INFO [StoreOpener-dbbcde27e8d425fcc5932f5841a8c339-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region dbbcde27e8d425fcc5932f5841a8c339 2024-11-10T22:37:35,397 INFO [StoreOpener-dbbcde27e8d425fcc5932f5841a8c339-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dbbcde27e8d425fcc5932f5841a8c339 columnFamilyName C 2024-11-10T22:37:35,397 DEBUG [StoreOpener-dbbcde27e8d425fcc5932f5841a8c339-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:37:35,398 INFO [StoreOpener-dbbcde27e8d425fcc5932f5841a8c339-1 {}] regionserver.HStore(327): Store=dbbcde27e8d425fcc5932f5841a8c339/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T22:37:35,398 DEBUG [RS_OPEN_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for dbbcde27e8d425fcc5932f5841a8c339 2024-11-10T22:37:35,399 DEBUG [RS_OPEN_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339 2024-11-10T22:37:35,400 DEBUG [RS_OPEN_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339 2024-11-10T22:37:35,402 DEBUG [RS_OPEN_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for dbbcde27e8d425fcc5932f5841a8c339 2024-11-10T22:37:35,402 DEBUG [RS_OPEN_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for dbbcde27e8d425fcc5932f5841a8c339 2024-11-10T22:37:35,403 DEBUG [RS_OPEN_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-10T22:37:35,406 DEBUG [RS_OPEN_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for dbbcde27e8d425fcc5932f5841a8c339 2024-11-10T22:37:35,409 DEBUG [RS_OPEN_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-10T22:37:35,410 INFO [RS_OPEN_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened dbbcde27e8d425fcc5932f5841a8c339; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74571216, jitterRate=0.11119771003723145}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-10T22:37:35,410 DEBUG [RS_OPEN_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for dbbcde27e8d425fcc5932f5841a8c339 2024-11-10T22:37:35,411 DEBUG [RS_OPEN_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for dbbcde27e8d425fcc5932f5841a8c339: Running coprocessor pre-open hook at 1731278255386Writing region info on filesystem at 1731278255386Initializing all the Stores at 1731278255388 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731278255388Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731278255388Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731278255388Cleaning up temporary data from old regions at 1731278255402 (+14 ms)Running coprocessor post-open hooks at 1731278255410 (+8 ms)Region opened successfully at 1731278255411 (+1 ms) 2024-11-10T22:37:35,413 INFO [RS_OPEN_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., pid=6, masterSystemTime=1731278255378 2024-11-10T22:37:35,417 DEBUG [RS_OPEN_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. 2024-11-10T22:37:35,417 INFO [RS_OPEN_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. 2024-11-10T22:37:35,418 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=dbbcde27e8d425fcc5932f5841a8c339, regionState=OPEN, openSeqNum=2, regionLocation=849d0d71310f,43753,1731278250663 2024-11-10T22:37:35,422 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 because future has completed 2024-11-10T22:37:35,427 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-10T22:37:35,427 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 in 200 msec 2024-11-10T22:37:35,431 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-10T22:37:35,431 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=dbbcde27e8d425fcc5932f5841a8c339, ASSIGN in 365 msec 2024-11-10T22:37:35,432 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-10T22:37:35,433 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731278255432"}]},"ts":"1731278255432"} 2024-11-10T22:37:35,436 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-10T22:37:35,437 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-10T22:37:35,440 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2450 sec 2024-11-10T22:37:36,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-10T22:37:36,375 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-10T22:37:36,381 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ad452c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:37:36,381 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849d0d71310f,43351,-1 for getting cluster id 2024-11-10T22:37:36,381 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T22:37:36,383 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59ccc382-681c-4202-a6b6-e337b1afc6c7' 2024-11-10T22:37:36,384 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T22:37:36,384 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59ccc382-681c-4202-a6b6-e337b1afc6c7" 2024-11-10T22:37:36,384 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@596504f1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:37:36,384 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849d0d71310f,43351,-1] 2024-11-10T22:37:36,384 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T22:37:36,385 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:37:36,386 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46954, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T22:37:36,388 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51f7e812, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:37:36,388 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T22:37:36,389 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849d0d71310f,43753,1731278250663, seqNum=-1] 2024-11-10T22:37:36,390 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T22:37:36,392 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43644, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T22:37:36,395 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-10T22:37:36,396 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 849d0d71310f,43351,1731278249979 2024-11-10T22:37:36,396 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@2702f94 2024-11-10T22:37:36,399 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-10T22:37:36,401 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46960, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-10T22:37:36,405 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19d419ec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:37:36,405 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849d0d71310f,43351,-1 for getting cluster id 2024-11-10T22:37:36,405 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T22:37:36,406 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59ccc382-681c-4202-a6b6-e337b1afc6c7' 2024-11-10T22:37:36,407 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T22:37:36,407 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59ccc382-681c-4202-a6b6-e337b1afc6c7" 2024-11-10T22:37:36,407 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f4c9506, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:37:36,407 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849d0d71310f,43351,-1] 2024-11-10T22:37:36,407 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T22:37:36,408 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:37:36,409 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46972, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T22:37:36,410 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36644156, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:37:36,411 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@130d4a2a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:37:36,411 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849d0d71310f,43351,-1 for getting cluster id 2024-11-10T22:37:36,411 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T22:37:36,412 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59ccc382-681c-4202-a6b6-e337b1afc6c7' 2024-11-10T22:37:36,413 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T22:37:36,413 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59ccc382-681c-4202-a6b6-e337b1afc6c7" 2024-11-10T22:37:36,413 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b2da332, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:37:36,413 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849d0d71310f,43351,-1] 2024-11-10T22:37:36,414 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T22:37:36,414 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:37:36,415 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46986, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T22:37:36,416 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e3687d5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:37:36,417 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59d35085, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:37:36,417 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849d0d71310f,43351,-1 for getting cluster id 2024-11-10T22:37:36,417 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T22:37:36,418 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59ccc382-681c-4202-a6b6-e337b1afc6c7' 2024-11-10T22:37:36,419 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T22:37:36,419 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59ccc382-681c-4202-a6b6-e337b1afc6c7" 2024-11-10T22:37:36,419 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3937ace3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:37:36,419 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849d0d71310f,43351,-1] 2024-11-10T22:37:36,419 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T22:37:36,420 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:37:36,421 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47014, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T22:37:36,422 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30bed14c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:37:36,423 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fb4b46b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:37:36,423 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849d0d71310f,43351,-1 for getting cluster id 2024-11-10T22:37:36,423 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T22:37:36,424 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59ccc382-681c-4202-a6b6-e337b1afc6c7' 2024-11-10T22:37:36,424 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T22:37:36,425 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59ccc382-681c-4202-a6b6-e337b1afc6c7" 2024-11-10T22:37:36,425 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b130fb1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:37:36,425 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849d0d71310f,43351,-1] 2024-11-10T22:37:36,425 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T22:37:36,425 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:37:36,427 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47028, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T22:37:36,428 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ce4b195, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:37:36,428 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a1cce95, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:37:36,428 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849d0d71310f,43351,-1 for getting cluster id 2024-11-10T22:37:36,429 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T22:37:36,429 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59ccc382-681c-4202-a6b6-e337b1afc6c7' 2024-11-10T22:37:36,430 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T22:37:36,430 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59ccc382-681c-4202-a6b6-e337b1afc6c7" 2024-11-10T22:37:36,430 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6371d69b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:37:36,430 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849d0d71310f,43351,-1] 2024-11-10T22:37:36,430 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T22:37:36,431 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:37:36,432 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47044, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T22:37:36,433 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ff3582c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:37:36,434 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3055dae7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:37:36,434 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849d0d71310f,43351,-1 for getting cluster id 2024-11-10T22:37:36,434 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T22:37:36,435 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59ccc382-681c-4202-a6b6-e337b1afc6c7' 2024-11-10T22:37:36,435 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T22:37:36,435 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59ccc382-681c-4202-a6b6-e337b1afc6c7" 2024-11-10T22:37:36,436 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2521eed1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:37:36,436 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849d0d71310f,43351,-1] 2024-11-10T22:37:36,436 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T22:37:36,437 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:37:36,438 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47070, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T22:37:36,439 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@359c1e82, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:37:36,440 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5238acdf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:37:36,440 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849d0d71310f,43351,-1 for getting cluster id 2024-11-10T22:37:36,440 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T22:37:36,441 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59ccc382-681c-4202-a6b6-e337b1afc6c7' 2024-11-10T22:37:36,441 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T22:37:36,441 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59ccc382-681c-4202-a6b6-e337b1afc6c7" 2024-11-10T22:37:36,442 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@526369d3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:37:36,442 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849d0d71310f,43351,-1] 2024-11-10T22:37:36,442 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T22:37:36,442 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:37:36,443 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47084, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T22:37:36,443 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@be2e700, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:37:36,445 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f4f2c18, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:37:36,445 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849d0d71310f,43351,-1 for getting cluster id 2024-11-10T22:37:36,445 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T22:37:36,446 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59ccc382-681c-4202-a6b6-e337b1afc6c7' 2024-11-10T22:37:36,446 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T22:37:36,447 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59ccc382-681c-4202-a6b6-e337b1afc6c7" 2024-11-10T22:37:36,447 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9f638da, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:37:36,447 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849d0d71310f,43351,-1] 2024-11-10T22:37:36,447 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T22:37:36,448 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:37:36,448 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47100, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T22:37:36,449 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66b367dd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:37:36,450 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6643136e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:37:36,450 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849d0d71310f,43351,-1 for getting cluster id 2024-11-10T22:37:36,450 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T22:37:36,451 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59ccc382-681c-4202-a6b6-e337b1afc6c7' 2024-11-10T22:37:36,451 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T22:37:36,451 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59ccc382-681c-4202-a6b6-e337b1afc6c7" 2024-11-10T22:37:36,451 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7aba1af6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:37:36,451 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849d0d71310f,43351,-1] 2024-11-10T22:37:36,451 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T22:37:36,452 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:37:36,452 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47116, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T22:37:36,453 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6348649e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:37:36,459 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T22:37:36,459 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T22:37:36,459 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T22:37:36,459 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T22:37:36,460 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T22:37:36,461 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849d0d71310f,43753,1731278250663, seqNum=-1] 2024-11-10T22:37:36,461 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849d0d71310f,43753,1731278250663, seqNum=-1] 2024-11-10T22:37:36,461 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849d0d71310f,43753,1731278250663, seqNum=-1] 2024-11-10T22:37:36,461 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T22:37:36,461 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T22:37:36,462 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T22:37:36,463 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43650, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T22:37:36,464 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43660, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T22:37:36,465 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T22:37:36,465 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43676, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T22:37:36,465 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849d0d71310f,43753,1731278250663, seqNum=-1] 2024-11-10T22:37:36,466 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T22:37:36,466 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849d0d71310f,43753,1731278250663, seqNum=-1] 2024-11-10T22:37:36,467 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T22:37:36,467 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849d0d71310f,43753,1731278250663, seqNum=-1] 2024-11-10T22:37:36,468 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43686, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T22:37:36,468 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T22:37:36,468 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43680, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T22:37:36,469 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43700, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T22:37:36,471 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2] 2024-11-10T22:37:36,471 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2] 2024-11-10T22:37:36,476 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2] 2024-11-10T22:37:36,476 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2] 2024-11-10T22:37:36,476 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2] 2024-11-10T22:37:36,477 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2] 2024-11-10T22:37:36,477 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T22:37:36,483 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849d0d71310f,43753,1731278250663, seqNum=-1] 2024-11-10T22:37:36,484 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T22:37:36,485 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T22:37:36,488 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43714, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T22:37:36,492 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849d0d71310f,43753,1731278250663, seqNum=-1] 2024-11-10T22:37:36,493 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T22:37:36,495 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43730, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T22:37:36,495 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2] 2024-11-10T22:37:36,498 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T22:37:36,499 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2] 2024-11-10T22:37:36,500 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849d0d71310f,43753,1731278250663, seqNum=-1] 2024-11-10T22:37:36,500 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T22:37:36,502 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43738, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T22:37:36,502 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T22:37:36,508 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2] 2024-11-10T22:37:36,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-11-10T22:37:36,512 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T22:37:36,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-10T22:37:36,515 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T22:37:36,517 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T22:37:36,548 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T22:37:36,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(8855): Flush requested on dbbcde27e8d425fcc5932f5841a8c339 2024-11-10T22:37:36,555 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing dbbcde27e8d425fcc5932f5841a8c339 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-10T22:37:36,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-10T22:37:36,672 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/A/1189fa75ce874ac6bb646298bd75533d is 50, key is test_row_0/A:col10/1731278256531/Put/seqid=0 2024-11-10T22:37:36,694 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-10T22:37:36,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. 2024-11-10T22:37:36,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. as already flushing 2024-11-10T22:37:36,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. 2024-11-10T22:37:36,700 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => dbbcde27e8d425fcc5932f5841a8c339, NAME => 'TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:37:36,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => dbbcde27e8d425fcc5932f5841a8c339, NAME => 'TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:37:36,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073741837_1013 (size=12001) 2024-11-10T22:37:36,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dbbcde27e8d425fcc5932f5841a8c339, NAME => 'TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dbbcde27e8d425fcc5932f5841a8c339, NAME => 'TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:37:36,744 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:36,744 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:36,746 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:36,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43730 deadline: 1731278316736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:36,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43650 deadline: 1731278316730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:36,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43680 deadline: 1731278316736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:36,752 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:36,752 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:36,752 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:36,753 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:36,753 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:36,754 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:36,754 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:36,754 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:36,754 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:36,761 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:36,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43700 deadline: 1731278316743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:36,762 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:36,762 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:36,762 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:36,769 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:36,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43676 deadline: 1731278316747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:36,770 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:36,770 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:36,770 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:36,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-10T22:37:36,867 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:36,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43650 deadline: 1731278316864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:36,868 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:36,868 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:36,868 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:36,871 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-10T22:37:36,872 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:36,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43680 deadline: 1731278316865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:36,877 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:36,877 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:36,873 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:36,877 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:36,877 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:36,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43730 deadline: 1731278316865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:36,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43676 deadline: 1731278316876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:36,878 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:36,878 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:36,879 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:36,879 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:36,879 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:36,879 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:36,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. 2024-11-10T22:37:36,879 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:36,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. as already flushing 2024-11-10T22:37:36,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. 2024-11-10T22:37:36,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43700 deadline: 1731278316877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:36,880 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:36,880 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:36,881 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:36,880 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => dbbcde27e8d425fcc5932f5841a8c339, NAME => 'TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:37:36,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => dbbcde27e8d425fcc5932f5841a8c339, NAME => 'TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:37:36,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dbbcde27e8d425fcc5932f5841a8c339, NAME => 'TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dbbcde27e8d425fcc5932f5841a8c339, NAME => 'TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:37:37,037 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-10T22:37:37,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. 2024-11-10T22:37:37,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. as already flushing 2024-11-10T22:37:37,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. 2024-11-10T22:37:37,039 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => dbbcde27e8d425fcc5932f5841a8c339, NAME => 'TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:37:37,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => dbbcde27e8d425fcc5932f5841a8c339, NAME => 'TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:37:37,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dbbcde27e8d425fcc5932f5841a8c339, NAME => 'TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dbbcde27e8d425fcc5932f5841a8c339, NAME => 'TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:37:37,074 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:37,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43650 deadline: 1731278317073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:37,075 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:37,076 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:37,076 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:37,086 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:37,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43730 deadline: 1731278317085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:37,087 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:37,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43680 deadline: 1731278317086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:37,087 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:37,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43676 deadline: 1731278317086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:37,088 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:37,088 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:37,088 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:37,089 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:37,089 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:37,089 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:37,089 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:37,089 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:37,090 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:37,096 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:37,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43700 deadline: 1731278317093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:37,098 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:37,098 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:37,098 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:37,111 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/A/1189fa75ce874ac6bb646298bd75533d 2024-11-10T22:37:37,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-10T22:37:37,197 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-10T22:37:37,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. 2024-11-10T22:37:37,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. as already flushing 2024-11-10T22:37:37,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. 2024-11-10T22:37:37,198 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => dbbcde27e8d425fcc5932f5841a8c339, NAME => 'TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:37:37,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => dbbcde27e8d425fcc5932f5841a8c339, NAME => 'TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:37:37,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dbbcde27e8d425fcc5932f5841a8c339, NAME => 'TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dbbcde27e8d425fcc5932f5841a8c339, NAME => 'TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:37:37,220 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/B/5cd0ad941b914803bea6c3d527bf4359 is 50, key is test_row_0/B:col10/1731278256531/Put/seqid=0 2024-11-10T22:37:37,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073741838_1014 (size=12001) 2024-11-10T22:37:37,354 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-10T22:37:37,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. 2024-11-10T22:37:37,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. as already flushing 2024-11-10T22:37:37,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. 2024-11-10T22:37:37,355 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => dbbcde27e8d425fcc5932f5841a8c339, NAME => 'TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:37:37,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => dbbcde27e8d425fcc5932f5841a8c339, NAME => 'TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:37:37,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dbbcde27e8d425fcc5932f5841a8c339, NAME => 'TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dbbcde27e8d425fcc5932f5841a8c339, NAME => 'TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:37:37,387 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:37,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43650 deadline: 1731278317383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:37,388 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:37,389 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:37,389 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:37,396 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:37,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43676 deadline: 1731278317393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:37,397 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:37,397 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:37,397 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:37,397 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:37,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43730 deadline: 1731278317394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:37,398 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:37,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43680 deadline: 1731278317394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:37,398 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:37,398 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:37,398 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:37,399 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:37,399 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:37,399 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:37,405 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:37,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43700 deadline: 1731278317404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:37,406 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:37,406 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:37,406 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:37,510 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-10T22:37:37,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. 2024-11-10T22:37:37,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. as already flushing 2024-11-10T22:37:37,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. 2024-11-10T22:37:37,511 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => dbbcde27e8d425fcc5932f5841a8c339, NAME => 'TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:37:37,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => dbbcde27e8d425fcc5932f5841a8c339, NAME => 'TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:37:37,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dbbcde27e8d425fcc5932f5841a8c339, NAME => 'TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dbbcde27e8d425fcc5932f5841a8c339, NAME => 'TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:37:37,645 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/B/5cd0ad941b914803bea6c3d527bf4359 2024-11-10T22:37:37,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-10T22:37:37,667 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-10T22:37:37,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. 2024-11-10T22:37:37,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. as already flushing 2024-11-10T22:37:37,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. 2024-11-10T22:37:37,667 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => dbbcde27e8d425fcc5932f5841a8c339, NAME => 'TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:37:37,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => dbbcde27e8d425fcc5932f5841a8c339, NAME => 'TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:37:37,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dbbcde27e8d425fcc5932f5841a8c339, NAME => 'TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dbbcde27e8d425fcc5932f5841a8c339, NAME => 'TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:37:37,696 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/C/b9fe603f49564bc099d73affdf19ee8a is 50, key is test_row_0/C:col10/1731278256531/Put/seqid=0 2024-11-10T22:37:37,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073741839_1015 (size=12001) 2024-11-10T22:37:37,719 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/C/b9fe603f49564bc099d73affdf19ee8a 2024-11-10T22:37:37,734 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/A/1189fa75ce874ac6bb646298bd75533d as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/1189fa75ce874ac6bb646298bd75533d 2024-11-10T22:37:37,749 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/1189fa75ce874ac6bb646298bd75533d, entries=150, sequenceid=13, filesize=11.7 K 2024-11-10T22:37:37,755 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/B/5cd0ad941b914803bea6c3d527bf4359 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/5cd0ad941b914803bea6c3d527bf4359 2024-11-10T22:37:37,766 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/5cd0ad941b914803bea6c3d527bf4359, entries=150, sequenceid=13, filesize=11.7 K 2024-11-10T22:37:37,768 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/C/b9fe603f49564bc099d73affdf19ee8a as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/b9fe603f49564bc099d73affdf19ee8a 2024-11-10T22:37:37,791 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/b9fe603f49564bc099d73affdf19ee8a, entries=150, sequenceid=13, filesize=11.7 K 2024-11-10T22:37:37,803 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=154.31 KB/158010 for dbbcde27e8d425fcc5932f5841a8c339 in 1248ms, sequenceid=13, compaction requested=false 2024-11-10T22:37:37,804 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-10T22:37:37,807 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for dbbcde27e8d425fcc5932f5841a8c339: 2024-11-10T22:37:37,825 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-10T22:37:37,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. 2024-11-10T22:37:37,826 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing dbbcde27e8d425fcc5932f5841a8c339 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-10T22:37:37,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/A/0c429c842d264d72ac93837d08e0e7fa is 50, key is test_row_0/A:col10/1731278256732/Put/seqid=0 2024-11-10T22:37:37,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073741840_1016 (size=12001) 2024-11-10T22:37:37,862 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=39 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/A/0c429c842d264d72ac93837d08e0e7fa 2024-11-10T22:37:37,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/B/c6535186dc4d483d87772535592b4b05 is 50, key is test_row_0/B:col10/1731278256732/Put/seqid=0 2024-11-10T22:37:37,915 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. as already flushing 2024-11-10T22:37:37,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(8855): Flush requested on dbbcde27e8d425fcc5932f5841a8c339 2024-11-10T22:37:37,935 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:37,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43676 deadline: 1731278317924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:37,936 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:37,937 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:37,937 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:37,937 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:37,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43700 deadline: 1731278317927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:37,939 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:37,939 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:37,939 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:37,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073741841_1017 (size=12001) 2024-11-10T22:37:37,950 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:37,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43650 deadline: 1731278317934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:37,952 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:37,952 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:37,952 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:37,952 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:37,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43730 deadline: 1731278317934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:37,953 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:37,953 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:37,953 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:37,953 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:37,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43680 deadline: 1731278317937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:37,955 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:37,955 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:37,955 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:38,051 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:38,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43700 deadline: 1731278318044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:38,052 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:38,052 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:38,052 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:38,052 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:38,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43676 deadline: 1731278318044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:38,053 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:38,053 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:38,054 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:38,064 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:38,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43680 deadline: 1731278318064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:38,066 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:38,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43650 deadline: 1731278318064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:38,066 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:38,066 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:38,066 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:38,067 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:38,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43730 deadline: 1731278318064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:38,067 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:38,067 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:38,067 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:38,067 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:38,067 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:38,067 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:38,267 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:38,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43700 deadline: 1731278318263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:38,268 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:38,269 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:38,269 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:38,269 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:38,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43676 deadline: 1731278318263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:38,270 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:38,270 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:38,270 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:38,274 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:38,274 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:38,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43680 deadline: 1731278318273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:38,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43650 deadline: 1731278318273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:38,275 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:38,275 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:38,275 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:38,275 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:38,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43730 deadline: 1731278318274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:38,276 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:38,276 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:38,276 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:38,276 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:38,277 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:38,277 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:38,345 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=39 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/B/c6535186dc4d483d87772535592b4b05 2024-11-10T22:37:38,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/C/255cc363969e44c4b682398a1c2fe1b6 is 50, key is test_row_0/C:col10/1731278256732/Put/seqid=0 2024-11-10T22:37:38,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073741842_1018 (size=12001) 2024-11-10T22:37:38,575 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:38,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43700 deadline: 1731278318573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:38,576 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:38,576 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:38,576 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:38,589 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:38,589 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:38,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43676 deadline: 1731278318583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:38,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43650 deadline: 1731278318584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:38,590 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:38,590 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:38,590 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:38,590 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:38,590 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:38,591 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:38,591 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:38,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43730 deadline: 1731278318585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:38,590 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:38,591 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:38,592 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:38,592 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:38,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43680 deadline: 1731278318584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:38,592 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:38,592 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:38,592 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:38,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-10T22:37:38,779 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=39 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/C/255cc363969e44c4b682398a1c2fe1b6 2024-11-10T22:37:38,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/A/0c429c842d264d72ac93837d08e0e7fa as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/0c429c842d264d72ac93837d08e0e7fa 2024-11-10T22:37:38,811 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/0c429c842d264d72ac93837d08e0e7fa, entries=150, sequenceid=39, filesize=11.7 K 2024-11-10T22:37:38,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/B/c6535186dc4d483d87772535592b4b05 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/c6535186dc4d483d87772535592b4b05 2024-11-10T22:37:38,843 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/c6535186dc4d483d87772535592b4b05, entries=150, sequenceid=39, filesize=11.7 K 2024-11-10T22:37:38,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-11-10T22:37:38,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/C/255cc363969e44c4b682398a1c2fe1b6 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/255cc363969e44c4b682398a1c2fe1b6 2024-11-10T22:37:38,858 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/255cc363969e44c4b682398a1c2fe1b6, entries=150, sequenceid=39, filesize=11.7 K 2024-11-10T22:37:38,862 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=53.67 KB/54960 for dbbcde27e8d425fcc5932f5841a8c339 in 1036ms, sequenceid=39, compaction requested=false 2024-11-10T22:37:38,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for dbbcde27e8d425fcc5932f5841a8c339: 2024-11-10T22:37:38,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. 2024-11-10T22:37:38,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-10T22:37:38,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-10T22:37:38,870 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-10T22:37:38,870 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3490 sec 2024-11-10T22:37:38,874 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 2.3670 sec 2024-11-10T22:37:39,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(8855): Flush requested on dbbcde27e8d425fcc5932f5841a8c339 2024-11-10T22:37:39,087 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T22:37:39,087 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing dbbcde27e8d425fcc5932f5841a8c339 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-10T22:37:39,098 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/A/404499702fea43888d35b759c7a99f88 is 50, key is test_row_0/A:col10/1731278259084/Put/seqid=0 2024-11-10T22:37:39,133 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073741843_1019 (size=12001) 2024-11-10T22:37:39,137 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/A/404499702fea43888d35b759c7a99f88 2024-11-10T22:37:39,159 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:39,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43680 deadline: 1731278319145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:39,160 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:39,160 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:39,161 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:39,160 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:39,161 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:39,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43700 deadline: 1731278319153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:39,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43650 deadline: 1731278319149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:39,162 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:39,162 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:39,162 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:39,163 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:39,163 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:39,163 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:39,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43676 deadline: 1731278319158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:39,163 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:39,164 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:39,164 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:39,164 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:39,165 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/B/ee95bc86ea854c3dbcd20fca6ad8dcde is 50, key is test_row_0/B:col10/1731278259084/Put/seqid=0 2024-11-10T22:37:39,167 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:39,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43730 deadline: 1731278319161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:39,168 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:39,168 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:39,168 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:39,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073741844_1020 (size=12001) 2024-11-10T22:37:39,192 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/B/ee95bc86ea854c3dbcd20fca6ad8dcde 2024-11-10T22:37:39,212 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/C/abbc714f137140b18a2e2ba4af9fb5dc is 50, key is test_row_0/C:col10/1731278259084/Put/seqid=0 2024-11-10T22:37:39,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073741845_1021 (size=12001) 2024-11-10T22:37:39,234 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/C/abbc714f137140b18a2e2ba4af9fb5dc 2024-11-10T22:37:39,247 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/A/404499702fea43888d35b759c7a99f88 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/404499702fea43888d35b759c7a99f88 2024-11-10T22:37:39,264 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/404499702fea43888d35b759c7a99f88, entries=150, sequenceid=51, filesize=11.7 K 2024-11-10T22:37:39,273 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/B/ee95bc86ea854c3dbcd20fca6ad8dcde as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/ee95bc86ea854c3dbcd20fca6ad8dcde 2024-11-10T22:37:39,276 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-10T22:37:39,276 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:39,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43680 deadline: 1731278319273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:39,277 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:39,277 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:39,277 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:39,281 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:39,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43650 deadline: 1731278319274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:39,283 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:39,283 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:39,283 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:39,283 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:39,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43676 deadline: 1731278319274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:39,284 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:39,284 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:39,284 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:39,285 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:39,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43730 deadline: 1731278319275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:39,286 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:39,286 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:39,286 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:39,286 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:39,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43700 deadline: 1731278319273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:39,287 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:39,287 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:39,287 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:39,290 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/ee95bc86ea854c3dbcd20fca6ad8dcde, entries=150, sequenceid=51, filesize=11.7 K 2024-11-10T22:37:39,292 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/C/abbc714f137140b18a2e2ba4af9fb5dc as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/abbc714f137140b18a2e2ba4af9fb5dc 2024-11-10T22:37:39,305 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/abbc714f137140b18a2e2ba4af9fb5dc, entries=150, sequenceid=51, filesize=11.7 K 2024-11-10T22:37:39,307 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for dbbcde27e8d425fcc5932f5841a8c339 in 220ms, sequenceid=51, compaction requested=true 2024-11-10T22:37:39,307 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for dbbcde27e8d425fcc5932f5841a8c339: 2024-11-10T22:37:39,316 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dbbcde27e8d425fcc5932f5841a8c339:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T22:37:39,318 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T22:37:39,318 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dbbcde27e8d425fcc5932f5841a8c339:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T22:37:39,319 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T22:37:39,319 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dbbcde27e8d425fcc5932f5841a8c339:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T22:37:39,319 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T22:37:39,322 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T22:37:39,322 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T22:37:39,325 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T22:37:39,328 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1541): dbbcde27e8d425fcc5932f5841a8c339/B is initiating minor compaction (all files) 2024-11-10T22:37:39,328 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T22:37:39,328 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HStore(1541): dbbcde27e8d425fcc5932f5841a8c339/A is initiating minor compaction (all files) 2024-11-10T22:37:39,329 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dbbcde27e8d425fcc5932f5841a8c339/B in TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. 2024-11-10T22:37:39,329 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dbbcde27e8d425fcc5932f5841a8c339/A in TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. 2024-11-10T22:37:39,329 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/1189fa75ce874ac6bb646298bd75533d, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/0c429c842d264d72ac93837d08e0e7fa, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/404499702fea43888d35b759c7a99f88] into tmpdir=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp, totalSize=35.2 K 2024-11-10T22:37:39,329 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/5cd0ad941b914803bea6c3d527bf4359, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/c6535186dc4d483d87772535592b4b05, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/ee95bc86ea854c3dbcd20fca6ad8dcde] into tmpdir=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp, totalSize=35.2 K 2024-11-10T22:37:39,331 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting 5cd0ad941b914803bea6c3d527bf4359, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731278256523 2024-11-10T22:37:39,337 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1189fa75ce874ac6bb646298bd75533d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731278256523 2024-11-10T22:37:39,337 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting c6535186dc4d483d87772535592b4b05, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=39, earliestPutTs=1731278256728 2024-11-10T22:37:39,340 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting ee95bc86ea854c3dbcd20fca6ad8dcde, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731278257928 2024-11-10T22:37:39,340 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0c429c842d264d72ac93837d08e0e7fa, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=39, earliestPutTs=1731278256728 2024-11-10T22:37:39,341 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 404499702fea43888d35b759c7a99f88, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731278257928 2024-11-10T22:37:39,355 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-10T22:37:39,377 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): dbbcde27e8d425fcc5932f5841a8c339#A#compaction#9 average throughput is 0.73 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T22:37:39,377 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): dbbcde27e8d425fcc5932f5841a8c339#B#compaction#10 average throughput is 0.82 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T22:37:39,379 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/A/30db2d12e41c44208f7fe2a15c2349d8 is 50, key is test_row_0/A:col10/1731278259084/Put/seqid=0 2024-11-10T22:37:39,379 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/B/4aa024b221a84c3295da743597124bd0 is 50, key is test_row_0/B:col10/1731278259084/Put/seqid=0 2024-11-10T22:37:39,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073741846_1022 (size=12104) 2024-11-10T22:37:39,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073741847_1023 (size=12104) 2024-11-10T22:37:39,415 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/B/4aa024b221a84c3295da743597124bd0 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/4aa024b221a84c3295da743597124bd0 2024-11-10T22:37:39,445 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dbbcde27e8d425fcc5932f5841a8c339/B of dbbcde27e8d425fcc5932f5841a8c339 into 4aa024b221a84c3295da743597124bd0(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T22:37:39,445 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dbbcde27e8d425fcc5932f5841a8c339: 2024-11-10T22:37:39,448 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., storeName=dbbcde27e8d425fcc5932f5841a8c339/B, priority=13, startTime=1731278259318; duration=0sec 2024-11-10T22:37:39,448 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T22:37:39,449 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dbbcde27e8d425fcc5932f5841a8c339:B 2024-11-10T22:37:39,449 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T22:37:39,452 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T22:37:39,452 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1541): dbbcde27e8d425fcc5932f5841a8c339/C is initiating minor compaction (all files) 2024-11-10T22:37:39,452 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dbbcde27e8d425fcc5932f5841a8c339/C in TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. 2024-11-10T22:37:39,452 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/b9fe603f49564bc099d73affdf19ee8a, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/255cc363969e44c4b682398a1c2fe1b6, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/abbc714f137140b18a2e2ba4af9fb5dc] into tmpdir=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp, totalSize=35.2 K 2024-11-10T22:37:39,453 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting b9fe603f49564bc099d73affdf19ee8a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731278256523 2024-11-10T22:37:39,455 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting 255cc363969e44c4b682398a1c2fe1b6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=39, earliestPutTs=1731278256728 2024-11-10T22:37:39,456 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting abbc714f137140b18a2e2ba4af9fb5dc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731278257928 2024-11-10T22:37:39,485 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): dbbcde27e8d425fcc5932f5841a8c339#C#compaction#11 average throughput is 0.82 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T22:37:39,486 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/C/56020c56435d4253b31b6f1b7d5ab67f is 50, key is test_row_0/C:col10/1731278259084/Put/seqid=0 2024-11-10T22:37:39,490 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T22:37:39,491 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing dbbcde27e8d425fcc5932f5841a8c339 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-10T22:37:39,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(8855): Flush requested on dbbcde27e8d425fcc5932f5841a8c339 2024-11-10T22:37:39,504 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/A/771a9c58cabc4d4fbd00595c0c7c0174 is 50, key is test_row_0/A:col10/1731278259157/Put/seqid=0 2024-11-10T22:37:39,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073741848_1024 (size=12104) 2024-11-10T22:37:39,524 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:39,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43650 deadline: 1731278319510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:39,525 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:39,525 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:39,525 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:39,525 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:39,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43680 deadline: 1731278319510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:39,526 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/C/56020c56435d4253b31b6f1b7d5ab67f as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/56020c56435d4253b31b6f1b7d5ab67f 2024-11-10T22:37:39,526 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:39,526 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:39,526 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:39,529 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:39,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43676 deadline: 1731278319518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:39,531 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:39,531 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:39,531 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:39,533 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:39,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43730 deadline: 1731278319522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:39,534 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:39,535 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:39,535 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:39,535 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:39,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073741849_1025 (size=14341) 2024-11-10T22:37:39,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43700 deadline: 1731278319525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:39,536 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:39,536 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:39,536 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:39,540 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/A/771a9c58cabc4d4fbd00595c0c7c0174 2024-11-10T22:37:39,544 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dbbcde27e8d425fcc5932f5841a8c339/C of dbbcde27e8d425fcc5932f5841a8c339 into 56020c56435d4253b31b6f1b7d5ab67f(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T22:37:39,545 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dbbcde27e8d425fcc5932f5841a8c339: 2024-11-10T22:37:39,545 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., storeName=dbbcde27e8d425fcc5932f5841a8c339/C, priority=13, startTime=1731278259319; duration=0sec 2024-11-10T22:37:39,545 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T22:37:39,545 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dbbcde27e8d425fcc5932f5841a8c339:C 2024-11-10T22:37:39,564 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/B/569791a5aa134c959326822d6dd5c0df is 50, key is test_row_0/B:col10/1731278259157/Put/seqid=0 2024-11-10T22:37:39,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073741850_1026 (size=12001) 2024-11-10T22:37:39,633 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:39,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43650 deadline: 1731278319632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:39,634 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:39,634 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:39,634 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:39,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43680 deadline: 1731278319633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:39,635 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:39,635 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:39,635 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:39,635 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:39,635 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:39,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43676 deadline: 1731278319633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:39,636 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:39,636 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:39,636 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:39,644 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:39,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43730 deadline: 1731278319643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:39,645 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:39,645 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:39,646 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:39,648 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:39,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43700 deadline: 1731278319643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:39,649 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:39,649 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:39,649 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:39,816 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/A/30db2d12e41c44208f7fe2a15c2349d8 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/30db2d12e41c44208f7fe2a15c2349d8 2024-11-10T22:37:39,830 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dbbcde27e8d425fcc5932f5841a8c339/A of dbbcde27e8d425fcc5932f5841a8c339 into 30db2d12e41c44208f7fe2a15c2349d8(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T22:37:39,830 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dbbcde27e8d425fcc5932f5841a8c339: 2024-11-10T22:37:39,830 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., storeName=dbbcde27e8d425fcc5932f5841a8c339/A, priority=13, startTime=1731278259309; duration=0sec 2024-11-10T22:37:39,830 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T22:37:39,830 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dbbcde27e8d425fcc5932f5841a8c339:A 2024-11-10T22:37:39,846 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:39,846 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:39,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43650 deadline: 1731278319843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:39,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43680 deadline: 1731278319843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:39,847 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:39,847 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:39,847 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:39,847 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:39,847 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:39,847 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:39,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43676 deadline: 1731278319843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:39,847 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:39,848 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:39,848 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:39,848 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:39,854 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:39,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43700 deadline: 1731278319853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:39,855 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:39,855 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:39,855 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:39,855 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:39,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43730 deadline: 1731278319853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:39,855 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:39,856 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:39,856 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:39,980 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/B/569791a5aa134c959326822d6dd5c0df 2024-11-10T22:37:40,008 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/C/7769fbdd99834f5f97f2cccd1f0537f6 is 50, key is test_row_0/C:col10/1731278259157/Put/seqid=0 2024-11-10T22:37:40,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073741851_1027 (size=12001) 2024-11-10T22:37:40,153 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:40,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43680 deadline: 1731278320152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:40,154 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:40,154 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:40,154 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:40,154 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:40,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43650 deadline: 1731278320153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:40,155 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:40,155 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:40,155 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:40,155 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:40,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43676 deadline: 1731278320153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:40,156 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:40,156 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:40,156 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:40,163 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:40,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43700 deadline: 1731278320162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:40,164 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:40,164 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:40,164 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:40,164 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:40,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43730 deadline: 1731278320162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:40,166 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:40,166 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:40,166 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:40,393 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-10T22:37:40,393 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-10T22:37:40,396 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-10T22:37:40,396 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-10T22:37:40,397 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-10T22:37:40,397 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-10T22:37:40,398 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-10T22:37:40,398 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-10T22:37:40,424 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/C/7769fbdd99834f5f97f2cccd1f0537f6 2024-11-10T22:37:40,440 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/A/771a9c58cabc4d4fbd00595c0c7c0174 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/771a9c58cabc4d4fbd00595c0c7c0174 2024-11-10T22:37:40,451 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/771a9c58cabc4d4fbd00595c0c7c0174, entries=200, sequenceid=77, filesize=14.0 K 2024-11-10T22:37:40,456 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/B/569791a5aa134c959326822d6dd5c0df as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/569791a5aa134c959326822d6dd5c0df 2024-11-10T22:37:40,468 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/569791a5aa134c959326822d6dd5c0df, entries=150, sequenceid=77, filesize=11.7 K 2024-11-10T22:37:40,470 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/C/7769fbdd99834f5f97f2cccd1f0537f6 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/7769fbdd99834f5f97f2cccd1f0537f6 2024-11-10T22:37:40,481 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/7769fbdd99834f5f97f2cccd1f0537f6, entries=150, sequenceid=77, filesize=11.7 K 2024-11-10T22:37:40,484 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for dbbcde27e8d425fcc5932f5841a8c339 in 994ms, sequenceid=77, compaction requested=false 2024-11-10T22:37:40,484 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for dbbcde27e8d425fcc5932f5841a8c339: 2024-11-10T22:37:40,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(8855): Flush requested on dbbcde27e8d425fcc5932f5841a8c339 2024-11-10T22:37:40,667 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T22:37:40,667 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing dbbcde27e8d425fcc5932f5841a8c339 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-10T22:37:40,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-10T22:37:40,674 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T22:37:40,676 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T22:37:40,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-11-10T22:37:40,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-10T22:37:40,687 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T22:37:40,687 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/A/6774ce93b7c34fb5a95373a28079588d is 50, key is test_row_0/A:col10/1731278260664/Put/seqid=0 2024-11-10T22:37:40,689 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T22:37:40,689 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T22:37:40,704 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073741852_1028 (size=12001) 2024-11-10T22:37:40,705 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/A/6774ce93b7c34fb5a95373a28079588d 2024-11-10T22:37:40,724 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/B/7db38536c6e9485896cc5532eefca58a is 50, key is test_row_0/B:col10/1731278260664/Put/seqid=0 2024-11-10T22:37:40,729 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:40,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43650 deadline: 1731278320713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:40,730 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:40,730 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:40,730 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:40,731 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:40,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43680 deadline: 1731278320719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:40,731 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:40,731 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:40,731 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:40,734 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:40,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43676 deadline: 1731278320726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:40,735 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:40,735 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:40,735 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:40,736 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:40,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43730 deadline: 1731278320727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:40,736 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:40,736 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:40,736 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:40,737 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:40,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43700 deadline: 1731278320730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:40,738 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:40,738 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:40,738 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:40,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073741853_1029 (size=12001) 2024-11-10T22:37:40,753 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/B/7db38536c6e9485896cc5532eefca58a 2024-11-10T22:37:40,770 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/C/b90640fb49c34a72adbca93d5a248290 is 50, key is test_row_0/C:col10/1731278260664/Put/seqid=0 2024-11-10T22:37:40,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-10T22:37:40,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073741854_1030 (size=12001) 2024-11-10T22:37:40,787 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/C/b90640fb49c34a72adbca93d5a248290 2024-11-10T22:37:40,797 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/A/6774ce93b7c34fb5a95373a28079588d as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/6774ce93b7c34fb5a95373a28079588d 2024-11-10T22:37:40,806 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/6774ce93b7c34fb5a95373a28079588d, entries=150, sequenceid=93, filesize=11.7 K 2024-11-10T22:37:40,808 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/B/7db38536c6e9485896cc5532eefca58a as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/7db38536c6e9485896cc5532eefca58a 2024-11-10T22:37:40,820 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/7db38536c6e9485896cc5532eefca58a, entries=150, sequenceid=93, filesize=11.7 K 2024-11-10T22:37:40,823 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/C/b90640fb49c34a72adbca93d5a248290 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/b90640fb49c34a72adbca93d5a248290 2024-11-10T22:37:40,833 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:40,833 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/b90640fb49c34a72adbca93d5a248290, entries=150, sequenceid=93, filesize=11.7 K 2024-11-10T22:37:40,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43650 deadline: 1731278320832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:40,834 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:40,834 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:40,834 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:40,835 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=134.18 KB/137400 for dbbcde27e8d425fcc5932f5841a8c339 in 168ms, sequenceid=93, compaction requested=true 2024-11-10T22:37:40,835 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for dbbcde27e8d425fcc5932f5841a8c339: 2024-11-10T22:37:40,835 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dbbcde27e8d425fcc5932f5841a8c339:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T22:37:40,835 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T22:37:40,835 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T22:37:40,835 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dbbcde27e8d425fcc5932f5841a8c339:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T22:37:40,835 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T22:37:40,836 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T22:37:40,836 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dbbcde27e8d425fcc5932f5841a8c339:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T22:37:40,836 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T22:37:40,839 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T22:37:40,839 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1541): dbbcde27e8d425fcc5932f5841a8c339/B is initiating minor compaction (all files) 2024-11-10T22:37:40,840 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dbbcde27e8d425fcc5932f5841a8c339/B in TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. 2024-11-10T22:37:40,840 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/4aa024b221a84c3295da743597124bd0, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/569791a5aa134c959326822d6dd5c0df, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/7db38536c6e9485896cc5532eefca58a] into tmpdir=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp, totalSize=35.3 K 2024-11-10T22:37:40,840 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38446 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T22:37:40,840 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HStore(1541): dbbcde27e8d425fcc5932f5841a8c339/A is initiating minor compaction (all files) 2024-11-10T22:37:40,841 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dbbcde27e8d425fcc5932f5841a8c339/A in TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. 2024-11-10T22:37:40,841 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/30db2d12e41c44208f7fe2a15c2349d8, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/771a9c58cabc4d4fbd00595c0c7c0174, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/6774ce93b7c34fb5a95373a28079588d] into tmpdir=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp, totalSize=37.5 K 2024-11-10T22:37:40,842 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting 4aa024b221a84c3295da743597124bd0, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731278257928 2024-11-10T22:37:40,842 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 30db2d12e41c44208f7fe2a15c2349d8, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731278257928 2024-11-10T22:37:40,844 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-10T22:37:40,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. 2024-11-10T22:37:40,845 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing dbbcde27e8d425fcc5932f5841a8c339 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-10T22:37:40,846 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting 569791a5aa134c959326822d6dd5c0df, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1731278259132 2024-11-10T22:37:40,846 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 771a9c58cabc4d4fbd00595c0c7c0174, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1731278259132 2024-11-10T22:37:40,847 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting 7db38536c6e9485896cc5532eefca58a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731278259521 2024-11-10T22:37:40,848 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6774ce93b7c34fb5a95373a28079588d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731278259521 2024-11-10T22:37:40,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(8855): Flush requested on dbbcde27e8d425fcc5932f5841a8c339 2024-11-10T22:37:40,851 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. as already flushing 2024-11-10T22:37:40,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/A/b1e8feacc0fc4dec8f5885dfaf3ec41f is 50, key is test_row_0/A:col10/1731278260723/Put/seqid=0 2024-11-10T22:37:40,875 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): dbbcde27e8d425fcc5932f5841a8c339#A#compaction#19 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T22:37:40,877 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/A/1b77b45ec7a842b3b3e8dc9f88ee636c is 50, key is test_row_0/A:col10/1731278260664/Put/seqid=0 2024-11-10T22:37:40,880 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:40,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43676 deadline: 1731278320869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:40,881 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:40,881 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:40,881 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:40,884 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:40,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43680 deadline: 1731278320877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:40,885 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:40,885 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:40,885 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:40,887 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:40,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43730 deadline: 1731278320880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:40,888 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:40,888 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:40,888 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:40,888 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:40,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43700 deadline: 1731278320880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:40,889 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:40,889 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:40,889 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:40,894 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): dbbcde27e8d425fcc5932f5841a8c339#B#compaction#20 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T22:37:40,894 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/B/dd73abff494245979542c0db14f502b3 is 50, key is test_row_0/B:col10/1731278260664/Put/seqid=0 2024-11-10T22:37:40,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073741855_1031 (size=12001) 2024-11-10T22:37:40,923 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/A/b1e8feacc0fc4dec8f5885dfaf3ec41f 2024-11-10T22:37:40,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073741856_1032 (size=12207) 2024-11-10T22:37:40,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073741857_1033 (size=12207) 2024-11-10T22:37:40,958 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/B/dd73abff494245979542c0db14f502b3 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/dd73abff494245979542c0db14f502b3 2024-11-10T22:37:40,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/B/ba7b11a739fc43c895618c1cfcdb9f7b is 50, key is test_row_0/B:col10/1731278260723/Put/seqid=0 2024-11-10T22:37:40,971 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dbbcde27e8d425fcc5932f5841a8c339/B of dbbcde27e8d425fcc5932f5841a8c339 into dd73abff494245979542c0db14f502b3(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T22:37:40,971 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dbbcde27e8d425fcc5932f5841a8c339: 2024-11-10T22:37:40,972 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., storeName=dbbcde27e8d425fcc5932f5841a8c339/B, priority=13, startTime=1731278260835; duration=0sec 2024-11-10T22:37:40,972 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T22:37:40,972 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dbbcde27e8d425fcc5932f5841a8c339:B 2024-11-10T22:37:40,972 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T22:37:40,976 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T22:37:40,977 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1541): dbbcde27e8d425fcc5932f5841a8c339/C is initiating minor compaction (all files) 2024-11-10T22:37:40,977 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dbbcde27e8d425fcc5932f5841a8c339/C in TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. 2024-11-10T22:37:40,978 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/56020c56435d4253b31b6f1b7d5ab67f, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/7769fbdd99834f5f97f2cccd1f0537f6, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/b90640fb49c34a72adbca93d5a248290] into tmpdir=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp, totalSize=35.3 K 2024-11-10T22:37:40,978 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting 56020c56435d4253b31b6f1b7d5ab67f, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731278257928 2024-11-10T22:37:40,979 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting 7769fbdd99834f5f97f2cccd1f0537f6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1731278259132 2024-11-10T22:37:40,980 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting b90640fb49c34a72adbca93d5a248290, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731278259521 2024-11-10T22:37:40,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073741858_1034 (size=12001) 2024-11-10T22:37:40,991 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/B/ba7b11a739fc43c895618c1cfcdb9f7b 2024-11-10T22:37:40,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-10T22:37:40,997 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:40,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43676 deadline: 1731278320993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:40,998 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:40,998 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:40,998 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:40,998 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:40,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43680 deadline: 1731278320994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:40,999 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:40,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43730 deadline: 1731278320994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:40,999 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:40,999 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:40,999 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:40,999 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:41,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43700 deadline: 1731278320994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:41,000 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:41,000 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:41,000 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:41,000 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:41,000 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:41,000 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:41,010 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): dbbcde27e8d425fcc5932f5841a8c339#C#compaction#22 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T22:37:41,013 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/C/37d709dc3b50494aacc564500dc132db is 50, key is test_row_0/C:col10/1731278260664/Put/seqid=0 2024-11-10T22:37:41,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/C/fd92350c73894ddb86b09d65c32b8961 is 50, key is test_row_0/C:col10/1731278260723/Put/seqid=0 2024-11-10T22:37:41,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073741860_1036 (size=12001) 2024-11-10T22:37:41,045 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:41,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43650 deadline: 1731278321043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:41,046 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/C/fd92350c73894ddb86b09d65c32b8961 2024-11-10T22:37:41,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073741859_1035 (size=12207) 2024-11-10T22:37:41,047 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:41,047 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:41,047 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:41,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/A/b1e8feacc0fc4dec8f5885dfaf3ec41f as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/b1e8feacc0fc4dec8f5885dfaf3ec41f 2024-11-10T22:37:41,064 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/C/37d709dc3b50494aacc564500dc132db as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/37d709dc3b50494aacc564500dc132db 2024-11-10T22:37:41,072 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/b1e8feacc0fc4dec8f5885dfaf3ec41f, entries=150, sequenceid=116, filesize=11.7 K 2024-11-10T22:37:41,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/B/ba7b11a739fc43c895618c1cfcdb9f7b as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/ba7b11a739fc43c895618c1cfcdb9f7b 2024-11-10T22:37:41,075 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dbbcde27e8d425fcc5932f5841a8c339/C of dbbcde27e8d425fcc5932f5841a8c339 into 37d709dc3b50494aacc564500dc132db(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T22:37:41,075 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dbbcde27e8d425fcc5932f5841a8c339: 2024-11-10T22:37:41,076 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., storeName=dbbcde27e8d425fcc5932f5841a8c339/C, priority=13, startTime=1731278260836; duration=0sec 2024-11-10T22:37:41,076 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T22:37:41,076 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dbbcde27e8d425fcc5932f5841a8c339:C 2024-11-10T22:37:41,084 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/ba7b11a739fc43c895618c1cfcdb9f7b, entries=150, sequenceid=116, filesize=11.7 K 2024-11-10T22:37:41,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/C/fd92350c73894ddb86b09d65c32b8961 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/fd92350c73894ddb86b09d65c32b8961 2024-11-10T22:37:41,095 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/fd92350c73894ddb86b09d65c32b8961, entries=150, sequenceid=116, filesize=11.7 K 2024-11-10T22:37:41,098 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=73.80 KB/75570 for dbbcde27e8d425fcc5932f5841a8c339 in 254ms, sequenceid=116, compaction requested=false 2024-11-10T22:37:41,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for dbbcde27e8d425fcc5932f5841a8c339: 2024-11-10T22:37:41,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. 2024-11-10T22:37:41,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-11-10T22:37:41,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-11-10T22:37:41,106 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-10T22:37:41,106 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 412 msec 2024-11-10T22:37:41,111 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 431 msec 2024-11-10T22:37:41,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(8855): Flush requested on dbbcde27e8d425fcc5932f5841a8c339 2024-11-10T22:37:41,207 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T22:37:41,207 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing dbbcde27e8d425fcc5932f5841a8c339 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-10T22:37:41,217 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/A/efb438a3fe764ceb913da2475ba4cc2d is 50, key is test_row_0/A:col10/1731278260878/Put/seqid=0 2024-11-10T22:37:41,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073741861_1037 (size=14541) 2024-11-10T22:37:41,247 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=135 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/A/efb438a3fe764ceb913da2475ba4cc2d 2024-11-10T22:37:41,253 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:41,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43676 deadline: 1731278321246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:41,254 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:41,254 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:41,254 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:41,254 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:41,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43730 deadline: 1731278321250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:41,255 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:41,255 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:41,255 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:41,257 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:41,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43700 deadline: 1731278321252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:41,259 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:41,259 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:41,259 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:41,259 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:41,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43680 deadline: 1731278321253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:41,260 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:41,260 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:41,260 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:41,263 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/B/c4f5736ecbb5464ca12a45bcdff82990 is 50, key is test_row_0/B:col10/1731278260878/Put/seqid=0 2024-11-10T22:37:41,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073741862_1038 (size=12151) 2024-11-10T22:37:41,286 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=135 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/B/c4f5736ecbb5464ca12a45bcdff82990 2024-11-10T22:37:41,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-10T22:37:41,303 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T22:37:41,303 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/C/c864390fb72e478c92394a123356081e is 50, key is test_row_0/C:col10/1731278260878/Put/seqid=0 2024-11-10T22:37:41,304 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T22:37:41,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-11-10T22:37:41,307 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T22:37:41,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-10T22:37:41,308 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T22:37:41,308 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T22:37:41,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073741863_1039 (size=12151) 2024-11-10T22:37:41,321 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=135 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/C/c864390fb72e478c92394a123356081e 2024-11-10T22:37:41,333 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/A/efb438a3fe764ceb913da2475ba4cc2d as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/efb438a3fe764ceb913da2475ba4cc2d 2024-11-10T22:37:41,351 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/A/1b77b45ec7a842b3b3e8dc9f88ee636c as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/1b77b45ec7a842b3b3e8dc9f88ee636c 2024-11-10T22:37:41,354 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/efb438a3fe764ceb913da2475ba4cc2d, entries=200, sequenceid=135, filesize=14.2 K 2024-11-10T22:37:41,356 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/B/c4f5736ecbb5464ca12a45bcdff82990 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/c4f5736ecbb5464ca12a45bcdff82990 2024-11-10T22:37:41,362 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:41,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43650 deadline: 1731278321352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:41,363 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:41,363 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:41,363 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:41,364 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:41,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43700 deadline: 1731278321363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:41,364 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:41,365 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:41,365 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:41,365 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:41,365 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:41,365 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dbbcde27e8d425fcc5932f5841a8c339/A of dbbcde27e8d425fcc5932f5841a8c339 into 1b77b45ec7a842b3b3e8dc9f88ee636c(size=11.9 K), total size for store is 37.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T22:37:41,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43676 deadline: 1731278321363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:41,365 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dbbcde27e8d425fcc5932f5841a8c339: 2024-11-10T22:37:41,366 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., storeName=dbbcde27e8d425fcc5932f5841a8c339/A, priority=13, startTime=1731278260835; duration=0sec 2024-11-10T22:37:41,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43730 deadline: 1731278321364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:41,366 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T22:37:41,366 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dbbcde27e8d425fcc5932f5841a8c339:A 2024-11-10T22:37:41,366 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:41,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43680 deadline: 1731278321364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:41,366 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:41,367 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:41,367 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:41,367 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:41,367 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:41,367 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:41,368 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:41,368 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:41,368 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:41,373 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/c4f5736ecbb5464ca12a45bcdff82990, entries=150, sequenceid=135, filesize=11.9 K 2024-11-10T22:37:41,375 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/C/c864390fb72e478c92394a123356081e as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/c864390fb72e478c92394a123356081e 2024-11-10T22:37:41,390 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/c864390fb72e478c92394a123356081e, entries=150, sequenceid=135, filesize=11.9 K 2024-11-10T22:37:41,392 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=114.05 KB/116790 for dbbcde27e8d425fcc5932f5841a8c339 in 185ms, sequenceid=135, compaction requested=true 2024-11-10T22:37:41,392 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for dbbcde27e8d425fcc5932f5841a8c339: 2024-11-10T22:37:41,392 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dbbcde27e8d425fcc5932f5841a8c339:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T22:37:41,392 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T22:37:41,392 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T22:37:41,392 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T22:37:41,393 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dbbcde27e8d425fcc5932f5841a8c339:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T22:37:41,393 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T22:37:41,393 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dbbcde27e8d425fcc5932f5841a8c339:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T22:37:41,393 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T22:37:41,394 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38749 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T22:37:41,394 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HStore(1541): dbbcde27e8d425fcc5932f5841a8c339/A is initiating minor compaction (all files) 2024-11-10T22:37:41,395 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dbbcde27e8d425fcc5932f5841a8c339/A in TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. 2024-11-10T22:37:41,395 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/1b77b45ec7a842b3b3e8dc9f88ee636c, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/b1e8feacc0fc4dec8f5885dfaf3ec41f, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/efb438a3fe764ceb913da2475ba4cc2d] into tmpdir=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp, totalSize=37.8 K 2024-11-10T22:37:41,395 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T22:37:41,395 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1541): dbbcde27e8d425fcc5932f5841a8c339/B is initiating minor compaction (all files) 2024-11-10T22:37:41,395 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dbbcde27e8d425fcc5932f5841a8c339/B in TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. 2024-11-10T22:37:41,396 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/dd73abff494245979542c0db14f502b3, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/ba7b11a739fc43c895618c1cfcdb9f7b, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/c4f5736ecbb5464ca12a45bcdff82990] into tmpdir=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp, totalSize=35.5 K 2024-11-10T22:37:41,396 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1b77b45ec7a842b3b3e8dc9f88ee636c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731278259521 2024-11-10T22:37:41,396 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting dd73abff494245979542c0db14f502b3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731278259521 2024-11-10T22:37:41,397 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting b1e8feacc0fc4dec8f5885dfaf3ec41f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731278260704 2024-11-10T22:37:41,397 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting ba7b11a739fc43c895618c1cfcdb9f7b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731278260704 2024-11-10T22:37:41,398 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting c4f5736ecbb5464ca12a45bcdff82990, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1731278260878 2024-11-10T22:37:41,398 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting efb438a3fe764ceb913da2475ba4cc2d, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1731278260878 2024-11-10T22:37:41,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-10T22:37:41,418 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): dbbcde27e8d425fcc5932f5841a8c339#B#compaction#27 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T22:37:41,419 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/B/abb0891289fd47348f33a00617065baf is 50, key is test_row_0/B:col10/1731278260878/Put/seqid=0 2024-11-10T22:37:41,426 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): dbbcde27e8d425fcc5932f5841a8c339#A#compaction#28 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T22:37:41,427 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/A/7773dcf022a942cb998aeaf04c624f2e is 50, key is test_row_0/A:col10/1731278260878/Put/seqid=0 2024-11-10T22:37:41,443 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073741864_1040 (size=12459) 2024-11-10T22:37:41,455 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/B/abb0891289fd47348f33a00617065baf as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/abb0891289fd47348f33a00617065baf 2024-11-10T22:37:41,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073741865_1041 (size=12459) 2024-11-10T22:37:41,463 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-10T22:37:41,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. 2024-11-10T22:37:41,464 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing dbbcde27e8d425fcc5932f5841a8c339 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-10T22:37:41,469 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dbbcde27e8d425fcc5932f5841a8c339/B of dbbcde27e8d425fcc5932f5841a8c339 into abb0891289fd47348f33a00617065baf(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T22:37:41,470 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dbbcde27e8d425fcc5932f5841a8c339: 2024-11-10T22:37:41,470 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., storeName=dbbcde27e8d425fcc5932f5841a8c339/B, priority=13, startTime=1731278261392; duration=0sec 2024-11-10T22:37:41,470 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T22:37:41,470 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dbbcde27e8d425fcc5932f5841a8c339:B 2024-11-10T22:37:41,470 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T22:37:41,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/A/3a4ed58ed2ad4752921be7d8438bc1df is 50, key is test_row_0/A:col10/1731278261243/Put/seqid=0 2024-11-10T22:37:41,476 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T22:37:41,476 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1541): dbbcde27e8d425fcc5932f5841a8c339/C is initiating minor compaction (all files) 2024-11-10T22:37:41,477 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dbbcde27e8d425fcc5932f5841a8c339/C in TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. 2024-11-10T22:37:41,477 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/37d709dc3b50494aacc564500dc132db, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/fd92350c73894ddb86b09d65c32b8961, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/c864390fb72e478c92394a123356081e] into tmpdir=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp, totalSize=35.5 K 2024-11-10T22:37:41,478 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting 37d709dc3b50494aacc564500dc132db, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731278259521 2024-11-10T22:37:41,479 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting fd92350c73894ddb86b09d65c32b8961, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731278260704 2024-11-10T22:37:41,480 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting c864390fb72e478c92394a123356081e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1731278260878 2024-11-10T22:37:41,500 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): dbbcde27e8d425fcc5932f5841a8c339#C#compaction#30 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T22:37:41,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073741866_1042 (size=12151) 2024-11-10T22:37:41,501 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/C/c46c520081464d26a8004e1bdb0f6a89 is 50, key is test_row_0/C:col10/1731278260878/Put/seqid=0 2024-11-10T22:37:41,503 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/A/3a4ed58ed2ad4752921be7d8438bc1df 2024-11-10T22:37:41,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/B/18e5803b2cec4f73964080a404e80f71 is 50, key is test_row_0/B:col10/1731278261243/Put/seqid=0 2024-11-10T22:37:41,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073741867_1043 (size=12459) 2024-11-10T22:37:41,535 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/C/c46c520081464d26a8004e1bdb0f6a89 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/c46c520081464d26a8004e1bdb0f6a89 2024-11-10T22:37:41,549 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dbbcde27e8d425fcc5932f5841a8c339/C of dbbcde27e8d425fcc5932f5841a8c339 into c46c520081464d26a8004e1bdb0f6a89(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T22:37:41,549 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dbbcde27e8d425fcc5932f5841a8c339: 2024-11-10T22:37:41,549 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., storeName=dbbcde27e8d425fcc5932f5841a8c339/C, priority=13, startTime=1731278261393; duration=0sec 2024-11-10T22:37:41,551 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T22:37:41,551 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dbbcde27e8d425fcc5932f5841a8c339:C 2024-11-10T22:37:41,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073741868_1044 (size=12151) 2024-11-10T22:37:41,555 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/B/18e5803b2cec4f73964080a404e80f71 2024-11-10T22:37:41,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(8855): Flush requested on dbbcde27e8d425fcc5932f5841a8c339 2024-11-10T22:37:41,575 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. as already flushing 2024-11-10T22:37:41,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/C/5ae989950a464c92862eb2a0763892a3 is 50, key is test_row_0/C:col10/1731278261243/Put/seqid=0 2024-11-10T22:37:41,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073741869_1045 (size=12151) 2024-11-10T22:37:41,595 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/C/5ae989950a464c92862eb2a0763892a3 2024-11-10T22:37:41,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/A/3a4ed58ed2ad4752921be7d8438bc1df as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/3a4ed58ed2ad4752921be7d8438bc1df 2024-11-10T22:37:41,609 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:41,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43680 deadline: 1731278321600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:41,610 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:41,610 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:41,610 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:41,611 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:41,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43730 deadline: 1731278321603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:41,612 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:41,612 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:41,612 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:41,616 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:41,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43676 deadline: 1731278321610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:41,617 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:41,617 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:41,617 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:41,617 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:41,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43700 deadline: 1731278321611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:41,618 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:41,618 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:41,618 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:41,620 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/3a4ed58ed2ad4752921be7d8438bc1df, entries=150, sequenceid=156, filesize=11.9 K 2024-11-10T22:37:41,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/B/18e5803b2cec4f73964080a404e80f71 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/18e5803b2cec4f73964080a404e80f71 2024-11-10T22:37:41,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-10T22:37:41,635 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/18e5803b2cec4f73964080a404e80f71, entries=150, sequenceid=156, filesize=11.9 K 2024-11-10T22:37:41,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/C/5ae989950a464c92862eb2a0763892a3 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/5ae989950a464c92862eb2a0763892a3 2024-11-10T22:37:41,651 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/5ae989950a464c92862eb2a0763892a3, entries=150, sequenceid=156, filesize=11.9 K 2024-11-10T22:37:41,653 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=93.93 KB/96180 for dbbcde27e8d425fcc5932f5841a8c339 in 189ms, sequenceid=156, compaction requested=false 2024-11-10T22:37:41,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for dbbcde27e8d425fcc5932f5841a8c339: 2024-11-10T22:37:41,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. 2024-11-10T22:37:41,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-11-10T22:37:41,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-11-10T22:37:41,660 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-10T22:37:41,660 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 348 msec 2024-11-10T22:37:41,665 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 358 msec 2024-11-10T22:37:41,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(8855): Flush requested on dbbcde27e8d425fcc5932f5841a8c339 2024-11-10T22:37:41,715 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T22:37:41,715 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing dbbcde27e8d425fcc5932f5841a8c339 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-10T22:37:41,726 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/A/1289798ebb01461089cd1f30d05781fb is 50, key is test_row_0/A:col10/1731278261597/Put/seqid=0 2024-11-10T22:37:41,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073741870_1046 (size=12151) 2024-11-10T22:37:41,737 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=176 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/A/1289798ebb01461089cd1f30d05781fb 2024-11-10T22:37:41,764 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:41,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43730 deadline: 1731278321763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:41,765 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:41,765 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:41,766 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:41,766 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:41,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43700 deadline: 1731278321764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:41,766 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:41,766 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:41,766 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:41,766 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:41,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43680 deadline: 1731278321764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:41,767 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:41,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43676 deadline: 1731278321764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:41,767 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:41,767 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:41,767 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:41,767 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:41,768 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:41,768 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:41,771 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/B/17a8fce7eec2421e973126a139aa71cb is 50, key is test_row_0/B:col10/1731278261597/Put/seqid=0 2024-11-10T22:37:41,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073741871_1047 (size=12151) 2024-11-10T22:37:41,787 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=176 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/B/17a8fce7eec2421e973126a139aa71cb 2024-11-10T22:37:41,810 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/C/59d76567a0484d10bdf7bce2a0afea46 is 50, key is test_row_0/C:col10/1731278261597/Put/seqid=0 2024-11-10T22:37:41,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073741872_1048 (size=12151) 2024-11-10T22:37:41,874 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:41,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43730 deadline: 1731278321874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:41,875 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:41,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43700 deadline: 1731278321874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:41,875 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:41,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43676 deadline: 1731278321874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:41,876 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:41,876 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:41,876 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:41,876 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:41,876 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:41,876 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:41,876 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:41,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43650 deadline: 1731278321875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:41,878 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:41,878 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:41,878 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:41,879 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/A/7773dcf022a942cb998aeaf04c624f2e as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/7773dcf022a942cb998aeaf04c624f2e 2024-11-10T22:37:41,880 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:41,880 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:41,880 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:41,876 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:41,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43680 deadline: 1731278321875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:41,881 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:41,881 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:41,881 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:41,890 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dbbcde27e8d425fcc5932f5841a8c339/A of dbbcde27e8d425fcc5932f5841a8c339 into 7773dcf022a942cb998aeaf04c624f2e(size=12.2 K), total size for store is 24.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T22:37:41,890 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dbbcde27e8d425fcc5932f5841a8c339: 2024-11-10T22:37:41,890 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., storeName=dbbcde27e8d425fcc5932f5841a8c339/A, priority=13, startTime=1731278261392; duration=0sec 2024-11-10T22:37:41,890 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T22:37:41,890 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dbbcde27e8d425fcc5932f5841a8c339:A 2024-11-10T22:37:41,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-10T22:37:41,933 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T22:37:41,934 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T22:37:41,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-11-10T22:37:41,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-10T22:37:41,938 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T22:37:41,939 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T22:37:41,940 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T22:37:42,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-10T22:37:42,084 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:42,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43730 deadline: 1731278322082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:42,086 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:42,086 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:42,086 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:42,086 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:42,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43700 deadline: 1731278322083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:42,086 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:42,086 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:42,086 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:42,086 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:42,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43676 deadline: 1731278322083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:42,087 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:42,087 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:42,087 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:42,095 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:42,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43680 deadline: 1731278322092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:42,096 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-10T22:37:42,096 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:42,096 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:42,096 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:42,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. 2024-11-10T22:37:42,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. as already flushing 2024-11-10T22:37:42,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. 2024-11-10T22:37:42,097 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => dbbcde27e8d425fcc5932f5841a8c339, NAME => 'TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:37:42,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => dbbcde27e8d425fcc5932f5841a8c339, NAME => 'TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:37:42,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dbbcde27e8d425fcc5932f5841a8c339, NAME => 'TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dbbcde27e8d425fcc5932f5841a8c339, NAME => 'TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:37:42,225 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=176 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/C/59d76567a0484d10bdf7bce2a0afea46 2024-11-10T22:37:42,239 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/A/1289798ebb01461089cd1f30d05781fb as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/1289798ebb01461089cd1f30d05781fb 2024-11-10T22:37:42,248 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/1289798ebb01461089cd1f30d05781fb, entries=150, sequenceid=176, filesize=11.9 K 2024-11-10T22:37:42,251 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-10T22:37:42,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. 2024-11-10T22:37:42,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. as already flushing 2024-11-10T22:37:42,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. 2024-11-10T22:37:42,252 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => dbbcde27e8d425fcc5932f5841a8c339, NAME => 'TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:37:42,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => dbbcde27e8d425fcc5932f5841a8c339, NAME => 'TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:37:42,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-10T22:37:42,253 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/B/17a8fce7eec2421e973126a139aa71cb as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/17a8fce7eec2421e973126a139aa71cb 2024-11-10T22:37:42,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dbbcde27e8d425fcc5932f5841a8c339, NAME => 'TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dbbcde27e8d425fcc5932f5841a8c339, NAME => 'TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:37:42,266 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/17a8fce7eec2421e973126a139aa71cb, entries=150, sequenceid=176, filesize=11.9 K 2024-11-10T22:37:42,268 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/C/59d76567a0484d10bdf7bce2a0afea46 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/59d76567a0484d10bdf7bce2a0afea46 2024-11-10T22:37:42,276 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/59d76567a0484d10bdf7bce2a0afea46, entries=150, sequenceid=176, filesize=11.9 K 2024-11-10T22:37:42,278 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=107.34 KB/109920 for dbbcde27e8d425fcc5932f5841a8c339 in 563ms, sequenceid=176, compaction requested=true 2024-11-10T22:37:42,278 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for dbbcde27e8d425fcc5932f5841a8c339: 2024-11-10T22:37:42,279 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dbbcde27e8d425fcc5932f5841a8c339:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T22:37:42,279 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T22:37:42,279 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T22:37:42,279 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dbbcde27e8d425fcc5932f5841a8c339:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T22:37:42,279 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T22:37:42,279 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dbbcde27e8d425fcc5932f5841a8c339:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T22:37:42,279 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T22:37:42,279 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-10T22:37:42,281 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T22:37:42,281 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HStore(1541): dbbcde27e8d425fcc5932f5841a8c339/A is initiating minor compaction (all files) 2024-11-10T22:37:42,281 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dbbcde27e8d425fcc5932f5841a8c339/A in TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. 2024-11-10T22:37:42,281 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/7773dcf022a942cb998aeaf04c624f2e, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/3a4ed58ed2ad4752921be7d8438bc1df, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/1289798ebb01461089cd1f30d05781fb] into tmpdir=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp, totalSize=35.9 K 2024-11-10T22:37:42,282 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T22:37:42,282 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1541): dbbcde27e8d425fcc5932f5841a8c339/B is initiating minor compaction (all files) 2024-11-10T22:37:42,282 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dbbcde27e8d425fcc5932f5841a8c339/B in TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. 2024-11-10T22:37:42,282 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/abb0891289fd47348f33a00617065baf, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/18e5803b2cec4f73964080a404e80f71, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/17a8fce7eec2421e973126a139aa71cb] into tmpdir=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp, totalSize=35.9 K 2024-11-10T22:37:42,283 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7773dcf022a942cb998aeaf04c624f2e, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1731278260878 2024-11-10T22:37:42,283 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting abb0891289fd47348f33a00617065baf, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1731278260878 2024-11-10T22:37:42,285 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting 18e5803b2cec4f73964080a404e80f71, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731278261243 2024-11-10T22:37:42,285 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3a4ed58ed2ad4752921be7d8438bc1df, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731278261243 2024-11-10T22:37:42,285 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting 17a8fce7eec2421e973126a139aa71cb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1731278261595 2024-11-10T22:37:42,286 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1289798ebb01461089cd1f30d05781fb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1731278261595 2024-11-10T22:37:42,300 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): dbbcde27e8d425fcc5932f5841a8c339#A#compaction#36 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T22:37:42,301 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/A/ef2cc67048ba48e38466d0cff4ed3acb is 50, key is test_row_0/A:col10/1731278261597/Put/seqid=0 2024-11-10T22:37:42,301 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): dbbcde27e8d425fcc5932f5841a8c339#B#compaction#37 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T22:37:42,302 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/B/3e61a0a7018b4fe1bbe6a91dcefebe63 is 50, key is test_row_0/B:col10/1731278261597/Put/seqid=0 2024-11-10T22:37:42,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073741873_1049 (size=12561) 2024-11-10T22:37:42,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073741874_1050 (size=12561) 2024-11-10T22:37:42,332 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/B/3e61a0a7018b4fe1bbe6a91dcefebe63 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/3e61a0a7018b4fe1bbe6a91dcefebe63 2024-11-10T22:37:42,341 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dbbcde27e8d425fcc5932f5841a8c339/B of dbbcde27e8d425fcc5932f5841a8c339 into 3e61a0a7018b4fe1bbe6a91dcefebe63(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T22:37:42,341 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dbbcde27e8d425fcc5932f5841a8c339: 2024-11-10T22:37:42,341 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., storeName=dbbcde27e8d425fcc5932f5841a8c339/B, priority=13, startTime=1731278262279; duration=0sec 2024-11-10T22:37:42,342 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T22:37:42,342 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dbbcde27e8d425fcc5932f5841a8c339:B 2024-11-10T22:37:42,342 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T22:37:42,344 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T22:37:42,344 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1541): dbbcde27e8d425fcc5932f5841a8c339/C is initiating minor compaction (all files) 2024-11-10T22:37:42,344 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dbbcde27e8d425fcc5932f5841a8c339/C in TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. 2024-11-10T22:37:42,344 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/c46c520081464d26a8004e1bdb0f6a89, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/5ae989950a464c92862eb2a0763892a3, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/59d76567a0484d10bdf7bce2a0afea46] into tmpdir=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp, totalSize=35.9 K 2024-11-10T22:37:42,345 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting c46c520081464d26a8004e1bdb0f6a89, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1731278260878 2024-11-10T22:37:42,346 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting 5ae989950a464c92862eb2a0763892a3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731278261243 2024-11-10T22:37:42,346 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting 59d76567a0484d10bdf7bce2a0afea46, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1731278261595 2024-11-10T22:37:42,366 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): dbbcde27e8d425fcc5932f5841a8c339#C#compaction#38 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T22:37:42,367 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/C/9c080a141597400dbd85f66394618e41 is 50, key is test_row_0/C:col10/1731278261597/Put/seqid=0 2024-11-10T22:37:42,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(8855): Flush requested on dbbcde27e8d425fcc5932f5841a8c339 2024-11-10T22:37:42,396 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073741875_1051 (size=12561) 2024-11-10T22:37:42,397 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T22:37:42,397 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing dbbcde27e8d425fcc5932f5841a8c339 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-10T22:37:42,406 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-10T22:37:42,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. 2024-11-10T22:37:42,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. as already flushing 2024-11-10T22:37:42,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. 2024-11-10T22:37:42,406 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => dbbcde27e8d425fcc5932f5841a8c339, NAME => 'TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:37:42,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => dbbcde27e8d425fcc5932f5841a8c339, NAME => 'TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:37:42,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dbbcde27e8d425fcc5932f5841a8c339, NAME => 'TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dbbcde27e8d425fcc5932f5841a8c339, NAME => 'TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:37:42,414 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/A/f22dacb7a9344f8cb06470098dd57b3e is 50, key is test_row_0/A:col10/1731278262394/Put/seqid=0 2024-11-10T22:37:42,417 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/C/9c080a141597400dbd85f66394618e41 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/9c080a141597400dbd85f66394618e41 2024-11-10T22:37:42,431 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dbbcde27e8d425fcc5932f5841a8c339/C of dbbcde27e8d425fcc5932f5841a8c339 into 9c080a141597400dbd85f66394618e41(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T22:37:42,431 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dbbcde27e8d425fcc5932f5841a8c339: 2024-11-10T22:37:42,431 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., storeName=dbbcde27e8d425fcc5932f5841a8c339/C, priority=13, startTime=1731278262279; duration=0sec 2024-11-10T22:37:42,431 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T22:37:42,431 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dbbcde27e8d425fcc5932f5841a8c339:C 2024-11-10T22:37:42,433 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:42,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43680 deadline: 1731278322424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:42,434 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:42,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43730 deadline: 1731278322432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:42,435 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:42,435 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:42,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073741876_1052 (size=12151) 2024-11-10T22:37:42,435 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:42,435 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:42,435 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:42,435 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:42,436 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=199 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/A/f22dacb7a9344f8cb06470098dd57b3e 2024-11-10T22:37:42,440 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:42,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43700 deadline: 1731278322434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:42,441 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:42,441 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:42,441 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:42,447 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:42,447 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/B/86a82ed3b0f849aabba7fffe2165c87b is 50, key is test_row_0/B:col10/1731278262394/Put/seqid=0 2024-11-10T22:37:42,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43676 deadline: 1731278322434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:42,447 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:42,448 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:42,448 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:42,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073741877_1053 (size=12151) 2024-11-10T22:37:42,544 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:42,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43680 deadline: 1731278322543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:42,545 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:42,545 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:42,545 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:42,545 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:42,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43730 deadline: 1731278322543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:42,546 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:42,546 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:42,546 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:42,556 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:42,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43700 deadline: 1731278322552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:42,557 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:42,557 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:42,557 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:42,557 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:42,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43676 deadline: 1731278322552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:42,558 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:42,558 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:42,558 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:42,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-10T22:37:42,563 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-10T22:37:42,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. 2024-11-10T22:37:42,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. as already flushing 2024-11-10T22:37:42,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. 2024-11-10T22:37:42,564 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => dbbcde27e8d425fcc5932f5841a8c339, NAME => 'TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:37:42,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => dbbcde27e8d425fcc5932f5841a8c339, NAME => 'TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:37:42,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dbbcde27e8d425fcc5932f5841a8c339, NAME => 'TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dbbcde27e8d425fcc5932f5841a8c339, NAME => 'TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:37:42,721 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-10T22:37:42,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. 2024-11-10T22:37:42,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. as already flushing 2024-11-10T22:37:42,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. 2024-11-10T22:37:42,721 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => dbbcde27e8d425fcc5932f5841a8c339, NAME => 'TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:37:42,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => dbbcde27e8d425fcc5932f5841a8c339, NAME => 'TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:37:42,723 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/A/ef2cc67048ba48e38466d0cff4ed3acb as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/ef2cc67048ba48e38466d0cff4ed3acb 2024-11-10T22:37:42,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dbbcde27e8d425fcc5932f5841a8c339, NAME => 'TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dbbcde27e8d425fcc5932f5841a8c339, NAME => 'TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:37:42,735 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dbbcde27e8d425fcc5932f5841a8c339/A of dbbcde27e8d425fcc5932f5841a8c339 into ef2cc67048ba48e38466d0cff4ed3acb(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T22:37:42,735 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dbbcde27e8d425fcc5932f5841a8c339: 2024-11-10T22:37:42,735 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., storeName=dbbcde27e8d425fcc5932f5841a8c339/A, priority=13, startTime=1731278262279; duration=0sec 2024-11-10T22:37:42,735 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T22:37:42,736 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dbbcde27e8d425fcc5932f5841a8c339:A 2024-11-10T22:37:42,753 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:42,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43680 deadline: 1731278322752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:42,754 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:42,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43730 deadline: 1731278322753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:42,755 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:42,755 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:42,755 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:42,755 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:42,755 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:42,755 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:42,764 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:42,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43700 deadline: 1731278322763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:42,765 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:42,765 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:42,765 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:42,765 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:42,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43676 deadline: 1731278322763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:42,766 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:42,766 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:42,766 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:42,854 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=199 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/B/86a82ed3b0f849aabba7fffe2165c87b 2024-11-10T22:37:42,870 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/C/2f7f3216e5a74c71bc88d4cafd575de7 is 50, key is test_row_0/C:col10/1731278262394/Put/seqid=0 2024-11-10T22:37:42,878 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-10T22:37:42,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. 2024-11-10T22:37:42,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. as already flushing 2024-11-10T22:37:42,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. 2024-11-10T22:37:42,879 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => dbbcde27e8d425fcc5932f5841a8c339, NAME => 'TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:37:42,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => dbbcde27e8d425fcc5932f5841a8c339, NAME => 'TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:37:42,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dbbcde27e8d425fcc5932f5841a8c339, NAME => 'TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dbbcde27e8d425fcc5932f5841a8c339, NAME => 'TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:37:42,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073741878_1054 (size=12151) 2024-11-10T22:37:42,891 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=199 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/C/2f7f3216e5a74c71bc88d4cafd575de7 2024-11-10T22:37:42,895 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:42,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43650 deadline: 1731278322893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:42,899 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 849d0d71310f,43753,1731278250663 for 'test_row_1' in dbbcde27e8d425fcc5932f5841a8c339 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2184 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-10T22:37:42,900 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:42,900 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:42,900 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:42,901 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/A/f22dacb7a9344f8cb06470098dd57b3e as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/f22dacb7a9344f8cb06470098dd57b3e 2024-11-10T22:37:42,911 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/f22dacb7a9344f8cb06470098dd57b3e, entries=150, sequenceid=199, filesize=11.9 K 2024-11-10T22:37:42,912 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/B/86a82ed3b0f849aabba7fffe2165c87b as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/86a82ed3b0f849aabba7fffe2165c87b 2024-11-10T22:37:42,921 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/86a82ed3b0f849aabba7fffe2165c87b, entries=150, sequenceid=199, filesize=11.9 K 2024-11-10T22:37:42,923 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/C/2f7f3216e5a74c71bc88d4cafd575de7 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/2f7f3216e5a74c71bc88d4cafd575de7 2024-11-10T22:37:42,932 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/2f7f3216e5a74c71bc88d4cafd575de7, entries=150, sequenceid=199, filesize=11.9 K 2024-11-10T22:37:42,933 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for dbbcde27e8d425fcc5932f5841a8c339 in 536ms, sequenceid=199, compaction requested=false 2024-11-10T22:37:42,934 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for dbbcde27e8d425fcc5932f5841a8c339: 2024-11-10T22:37:43,035 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-10T22:37:43,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. 2024-11-10T22:37:43,035 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing dbbcde27e8d425fcc5932f5841a8c339 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-10T22:37:43,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/A/8a3d6180dc9947098b88da50e9ed8ebf is 50, key is test_row_0/A:col10/1731278262431/Put/seqid=0 2024-11-10T22:37:43,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073741879_1055 (size=12151) 2024-11-10T22:37:43,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(8855): Flush requested on dbbcde27e8d425fcc5932f5841a8c339 2024-11-10T22:37:43,065 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. as already flushing 2024-11-10T22:37:43,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-10T22:37:43,096 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:43,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43700 deadline: 1731278323091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:43,097 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:43,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43676 deadline: 1731278323092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:43,098 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:43,098 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:43,098 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:43,098 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:43,099 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:43,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43680 deadline: 1731278323094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:43,099 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:43,099 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:43,099 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:43,099 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:43,099 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:43,099 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:43,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43730 deadline: 1731278323096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:43,100 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:43,100 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:43,100 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:43,208 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:43,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43676 deadline: 1731278323203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:43,209 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:43,209 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:43,209 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:43,209 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:43,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43700 deadline: 1731278323203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:43,210 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:43,210 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:43,210 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:43,210 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:43,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43680 deadline: 1731278323203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:43,210 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:43,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43730 deadline: 1731278323204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:43,211 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:43,211 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:43,211 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:43,211 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:43,211 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:43,211 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:43,416 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:43,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43676 deadline: 1731278323412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:43,417 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:43,417 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:43,417 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:43,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43700 deadline: 1731278323413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:43,417 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:43,418 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:43,418 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:43,418 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:43,422 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:43,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43680 deadline: 1731278323422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:43,423 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:43,423 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:43,423 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:43,424 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:43,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43730 deadline: 1731278323422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:43,426 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:43,426 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:43,426 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:43,456 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=216 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/A/8a3d6180dc9947098b88da50e9ed8ebf 2024-11-10T22:37:43,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/B/b37c539d162346e9af554d094255a24c is 50, key is test_row_0/B:col10/1731278262431/Put/seqid=0 2024-11-10T22:37:43,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073741880_1056 (size=12151) 2024-11-10T22:37:43,491 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=216 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/B/b37c539d162346e9af554d094255a24c 2024-11-10T22:37:43,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/C/efbf8d6c171d45b08fa5e22a765a3098 is 50, key is test_row_0/C:col10/1731278262431/Put/seqid=0 2024-11-10T22:37:43,547 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073741881_1057 (size=12151) 2024-11-10T22:37:43,726 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:43,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43676 deadline: 1731278323723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:43,727 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:43,727 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:43,727 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:43,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43700 deadline: 1731278323723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:43,727 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:43,728 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:43,728 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:43,728 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:43,733 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:43,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43680 deadline: 1731278323732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:43,734 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:43,733 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:43,734 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:43,734 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:43,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43730 deadline: 1731278323733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:43,734 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:43,734 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:43,734 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:43,949 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=216 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/C/efbf8d6c171d45b08fa5e22a765a3098 2024-11-10T22:37:43,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/A/8a3d6180dc9947098b88da50e9ed8ebf as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/8a3d6180dc9947098b88da50e9ed8ebf 2024-11-10T22:37:43,964 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/8a3d6180dc9947098b88da50e9ed8ebf, entries=150, sequenceid=216, filesize=11.9 K 2024-11-10T22:37:43,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/B/b37c539d162346e9af554d094255a24c as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/b37c539d162346e9af554d094255a24c 2024-11-10T22:37:43,973 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/b37c539d162346e9af554d094255a24c, entries=150, sequenceid=216, filesize=11.9 K 2024-11-10T22:37:43,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/C/efbf8d6c171d45b08fa5e22a765a3098 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/efbf8d6c171d45b08fa5e22a765a3098 2024-11-10T22:37:43,984 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/efbf8d6c171d45b08fa5e22a765a3098, entries=150, sequenceid=216, filesize=11.9 K 2024-11-10T22:37:43,986 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for dbbcde27e8d425fcc5932f5841a8c339 in 951ms, sequenceid=216, compaction requested=true 2024-11-10T22:37:43,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for dbbcde27e8d425fcc5932f5841a8c339: 2024-11-10T22:37:43,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. 2024-11-10T22:37:43,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-11-10T22:37:43,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-11-10T22:37:43,992 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-10T22:37:43,992 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0500 sec 2024-11-10T22:37:43,996 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 2.0600 sec 2024-11-10T22:37:44,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-10T22:37:44,082 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T22:37:44,083 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T22:37:44,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-11-10T22:37:44,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-10T22:37:44,086 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T22:37:44,087 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T22:37:44,087 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T22:37:44,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-10T22:37:44,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(8855): Flush requested on dbbcde27e8d425fcc5932f5841a8c339 2024-11-10T22:37:44,236 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T22:37:44,237 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing dbbcde27e8d425fcc5932f5841a8c339 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-10T22:37:44,243 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-10T22:37:44,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. 2024-11-10T22:37:44,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. as already flushing 2024-11-10T22:37:44,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. 2024-11-10T22:37:44,244 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/A/8a3546d820424d87a75ed1e02e82c478 is 50, key is test_row_0/A:col10/1731278263092/Put/seqid=0 2024-11-10T22:37:44,244 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => dbbcde27e8d425fcc5932f5841a8c339, NAME => 'TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:37:44,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => dbbcde27e8d425fcc5932f5841a8c339, NAME => 'TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:37:44,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dbbcde27e8d425fcc5932f5841a8c339, NAME => 'TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dbbcde27e8d425fcc5932f5841a8c339, NAME => 'TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:37:44,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073741882_1058 (size=14541) 2024-11-10T22:37:44,267 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:44,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43680 deadline: 1731278324260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:44,271 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:44,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43700 deadline: 1731278324266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:44,272 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:44,272 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:44,272 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:44,273 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:44,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43730 deadline: 1731278324267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:44,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43676 deadline: 1731278324268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:44,273 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:44,273 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:44,273 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:44,273 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:44,274 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:44,274 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:44,274 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:44,276 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:44,276 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:44,276 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:44,276 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/A/8a3546d820424d87a75ed1e02e82c478 2024-11-10T22:37:44,289 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/B/993ae27b597f4858af80e4559f6bca2a is 50, key is test_row_0/B:col10/1731278263092/Put/seqid=0 2024-11-10T22:37:44,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073741883_1059 (size=12151) 2024-11-10T22:37:44,299 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/B/993ae27b597f4858af80e4559f6bca2a 2024-11-10T22:37:44,312 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/C/9d4c5f9d3ec94449b3a78127adcd1c6d is 50, key is test_row_0/C:col10/1731278263092/Put/seqid=0 2024-11-10T22:37:44,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073741884_1060 (size=12151) 2024-11-10T22:37:44,383 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:44,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43700 deadline: 1731278324383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:44,384 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:44,384 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:44,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43676 deadline: 1731278324383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:44,384 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:44,384 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:44,384 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:44,385 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:44,385 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:44,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43730 deadline: 1731278324383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:44,385 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:44,385 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:44,385 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:44,385 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:44,385 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:44,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43680 deadline: 1731278324383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:44,386 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:44,386 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:44,386 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:44,399 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-10T22:37:44,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. 2024-11-10T22:37:44,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. as already flushing 2024-11-10T22:37:44,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. 2024-11-10T22:37:44,400 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => dbbcde27e8d425fcc5932f5841a8c339, NAME => 'TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:37:44,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => dbbcde27e8d425fcc5932f5841a8c339, NAME => 'TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:37:44,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dbbcde27e8d425fcc5932f5841a8c339, NAME => 'TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dbbcde27e8d425fcc5932f5841a8c339, NAME => 'TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:37:44,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-10T22:37:44,554 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-10T22:37:44,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. 2024-11-10T22:37:44,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. as already flushing 2024-11-10T22:37:44,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. 2024-11-10T22:37:44,554 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => dbbcde27e8d425fcc5932f5841a8c339, NAME => 'TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:37:44,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => dbbcde27e8d425fcc5932f5841a8c339, NAME => 'TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:37:44,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dbbcde27e8d425fcc5932f5841a8c339, NAME => 'TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dbbcde27e8d425fcc5932f5841a8c339, NAME => 'TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:37:44,594 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:44,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43700 deadline: 1731278324593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:44,594 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:44,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43676 deadline: 1731278324593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:44,595 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:44,595 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:44,595 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:44,595 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:44,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43730 deadline: 1731278324593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:44,595 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:44,595 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:44,595 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:44,595 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:44,595 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:44,595 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:44,595 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:44,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43680 deadline: 1731278324593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:44,596 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:44,596 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:44,596 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:44,709 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-10T22:37:44,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. 2024-11-10T22:37:44,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. as already flushing 2024-11-10T22:37:44,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. 2024-11-10T22:37:44,710 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => dbbcde27e8d425fcc5932f5841a8c339, NAME => 'TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:37:44,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => dbbcde27e8d425fcc5932f5841a8c339, NAME => 'TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:37:44,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dbbcde27e8d425fcc5932f5841a8c339, NAME => 'TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dbbcde27e8d425fcc5932f5841a8c339, NAME => 'TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:37:44,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-10T22:37:44,724 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/C/9d4c5f9d3ec94449b3a78127adcd1c6d 2024-11-10T22:37:44,731 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/A/8a3546d820424d87a75ed1e02e82c478 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/8a3546d820424d87a75ed1e02e82c478 2024-11-10T22:37:44,739 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/8a3546d820424d87a75ed1e02e82c478, entries=200, sequenceid=238, filesize=14.2 K 2024-11-10T22:37:44,740 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/B/993ae27b597f4858af80e4559f6bca2a as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/993ae27b597f4858af80e4559f6bca2a 2024-11-10T22:37:44,745 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/993ae27b597f4858af80e4559f6bca2a, entries=150, sequenceid=238, filesize=11.9 K 2024-11-10T22:37:44,746 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/C/9d4c5f9d3ec94449b3a78127adcd1c6d as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/9d4c5f9d3ec94449b3a78127adcd1c6d 2024-11-10T22:37:44,752 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/9d4c5f9d3ec94449b3a78127adcd1c6d, entries=150, sequenceid=238, filesize=11.9 K 2024-11-10T22:37:44,753 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for dbbcde27e8d425fcc5932f5841a8c339 in 517ms, sequenceid=238, compaction requested=true 2024-11-10T22:37:44,753 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for dbbcde27e8d425fcc5932f5841a8c339: 2024-11-10T22:37:44,754 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dbbcde27e8d425fcc5932f5841a8c339:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T22:37:44,754 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T22:37:44,754 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T22:37:44,754 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dbbcde27e8d425fcc5932f5841a8c339:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T22:37:44,754 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T22:37:44,754 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T22:37:44,754 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dbbcde27e8d425fcc5932f5841a8c339:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T22:37:44,754 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T22:37:44,755 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 51404 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T22:37:44,756 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HStore(1541): dbbcde27e8d425fcc5932f5841a8c339/A is initiating minor compaction (all files) 2024-11-10T22:37:44,756 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dbbcde27e8d425fcc5932f5841a8c339/A in TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. 2024-11-10T22:37:44,756 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/ef2cc67048ba48e38466d0cff4ed3acb, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/f22dacb7a9344f8cb06470098dd57b3e, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/8a3d6180dc9947098b88da50e9ed8ebf, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/8a3546d820424d87a75ed1e02e82c478] into tmpdir=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp, totalSize=50.2 K 2024-11-10T22:37:44,756 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49014 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T22:37:44,756 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting ef2cc67048ba48e38466d0cff4ed3acb, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1731278261595 2024-11-10T22:37:44,757 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1541): dbbcde27e8d425fcc5932f5841a8c339/B is initiating minor compaction (all files) 2024-11-10T22:37:44,757 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dbbcde27e8d425fcc5932f5841a8c339/B in TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. 2024-11-10T22:37:44,757 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/3e61a0a7018b4fe1bbe6a91dcefebe63, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/86a82ed3b0f849aabba7fffe2165c87b, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/b37c539d162346e9af554d094255a24c, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/993ae27b597f4858af80e4559f6bca2a] into tmpdir=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp, totalSize=47.9 K 2024-11-10T22:37:44,757 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting f22dacb7a9344f8cb06470098dd57b3e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1731278261746 2024-11-10T22:37:44,757 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting 3e61a0a7018b4fe1bbe6a91dcefebe63, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1731278261595 2024-11-10T22:37:44,757 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8a3d6180dc9947098b88da50e9ed8ebf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1731278262419 2024-11-10T22:37:44,758 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting 86a82ed3b0f849aabba7fffe2165c87b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1731278261746 2024-11-10T22:37:44,758 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8a3546d820424d87a75ed1e02e82c478, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731278263090 2024-11-10T22:37:44,758 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting b37c539d162346e9af554d094255a24c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1731278262419 2024-11-10T22:37:44,759 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting 993ae27b597f4858af80e4559f6bca2a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731278263092 2024-11-10T22:37:44,784 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): dbbcde27e8d425fcc5932f5841a8c339#A#compaction#48 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T22:37:44,785 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/A/7ed846158b1e43c3880b1358d6ba8941 is 50, key is test_row_0/A:col10/1731278263092/Put/seqid=0 2024-11-10T22:37:44,786 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): dbbcde27e8d425fcc5932f5841a8c339#B#compaction#49 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T22:37:44,787 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/B/f9694985fbdd4980a05aae5d3b026c60 is 50, key is test_row_0/B:col10/1731278263092/Put/seqid=0 2024-11-10T22:37:44,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073741885_1061 (size=12697) 2024-11-10T22:37:44,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073741886_1062 (size=12697) 2024-11-10T22:37:44,864 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-10T22:37:44,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. 2024-11-10T22:37:44,865 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing dbbcde27e8d425fcc5932f5841a8c339 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-10T22:37:44,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/A/32d409df8d524c90821cf8dab8e5f762 is 50, key is test_row_0/A:col10/1731278264249/Put/seqid=0 2024-11-10T22:37:44,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073741887_1063 (size=12151) 2024-11-10T22:37:44,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(8855): Flush requested on dbbcde27e8d425fcc5932f5841a8c339 2024-11-10T22:37:44,906 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. as already flushing 2024-11-10T22:37:44,939 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:44,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43650 deadline: 1731278324932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:44,940 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:44,940 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:44,940 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:44,940 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:44,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43676 deadline: 1731278324932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:44,941 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:44,941 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:44,941 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:44,941 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:44,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43730 deadline: 1731278324934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:44,941 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:44,941 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:44,941 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:44,941 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:44,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43700 deadline: 1731278324936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:44,942 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:44,942 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:44,942 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:44,944 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:44,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43680 deadline: 1731278324939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:44,944 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:44,944 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:44,945 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:45,044 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:45,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43650 deadline: 1731278325043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:45,045 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:45,045 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:45,045 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:45,045 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:45,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43676 deadline: 1731278325043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:45,046 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:45,046 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:45,046 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:45,056 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:45,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43730 deadline: 1731278325053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:45,056 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:45,056 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:45,057 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:45,056 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:45,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43700 deadline: 1731278325053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:45,057 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:45,057 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:45,057 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:45,057 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:45,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43680 deadline: 1731278325053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:45,058 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:45,058 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:45,058 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:45,211 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/A/7ed846158b1e43c3880b1358d6ba8941 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/7ed846158b1e43c3880b1358d6ba8941 2024-11-10T22:37:45,212 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/B/f9694985fbdd4980a05aae5d3b026c60 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/f9694985fbdd4980a05aae5d3b026c60 2024-11-10T22:37:45,221 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in dbbcde27e8d425fcc5932f5841a8c339/A of dbbcde27e8d425fcc5932f5841a8c339 into 7ed846158b1e43c3880b1358d6ba8941(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T22:37:45,221 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in dbbcde27e8d425fcc5932f5841a8c339/B of dbbcde27e8d425fcc5932f5841a8c339 into f9694985fbdd4980a05aae5d3b026c60(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T22:37:45,221 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dbbcde27e8d425fcc5932f5841a8c339: 2024-11-10T22:37:45,221 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dbbcde27e8d425fcc5932f5841a8c339: 2024-11-10T22:37:45,221 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., storeName=dbbcde27e8d425fcc5932f5841a8c339/B, priority=12, startTime=1731278264754; duration=0sec 2024-11-10T22:37:45,221 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., storeName=dbbcde27e8d425fcc5932f5841a8c339/A, priority=12, startTime=1731278264753; duration=0sec 2024-11-10T22:37:45,221 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T22:37:45,221 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dbbcde27e8d425fcc5932f5841a8c339:B 2024-11-10T22:37:45,221 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T22:37:45,221 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dbbcde27e8d425fcc5932f5841a8c339:A 2024-11-10T22:37:45,221 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T22:37:45,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-10T22:37:45,225 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49014 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T22:37:45,226 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1541): dbbcde27e8d425fcc5932f5841a8c339/C is initiating minor compaction (all files) 2024-11-10T22:37:45,226 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dbbcde27e8d425fcc5932f5841a8c339/C in TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. 2024-11-10T22:37:45,226 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/9c080a141597400dbd85f66394618e41, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/2f7f3216e5a74c71bc88d4cafd575de7, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/efbf8d6c171d45b08fa5e22a765a3098, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/9d4c5f9d3ec94449b3a78127adcd1c6d] into tmpdir=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp, totalSize=47.9 K 2024-11-10T22:37:45,227 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting 9c080a141597400dbd85f66394618e41, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1731278261595 2024-11-10T22:37:45,227 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting 2f7f3216e5a74c71bc88d4cafd575de7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1731278261746 2024-11-10T22:37:45,228 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting efbf8d6c171d45b08fa5e22a765a3098, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1731278262419 2024-11-10T22:37:45,228 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting 9d4c5f9d3ec94449b3a78127adcd1c6d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731278263092 2024-11-10T22:37:45,238 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): dbbcde27e8d425fcc5932f5841a8c339#C#compaction#51 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T22:37:45,240 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/C/c3bc94638ef44755b83a5c137a1b4f2a is 50, key is test_row_0/C:col10/1731278263092/Put/seqid=0 2024-11-10T22:37:45,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073741888_1064 (size=12697) 2024-11-10T22:37:45,255 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:45,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43650 deadline: 1731278325253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:45,256 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:45,256 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:45,256 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:45,256 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:45,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43676 deadline: 1731278325253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:45,257 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:45,257 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:45,257 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:45,267 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:45,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43730 deadline: 1731278325262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:45,268 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:45,268 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:45,268 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:45,268 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:45,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43700 deadline: 1731278325263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:45,269 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:45,269 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:45,269 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:45,269 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:45,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43680 deadline: 1731278325263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:45,269 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:45,269 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:45,270 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:45,278 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=252 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/A/32d409df8d524c90821cf8dab8e5f762 2024-11-10T22:37:45,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/B/a9e5790d43df46ac94689c886f408356 is 50, key is test_row_0/B:col10/1731278264249/Put/seqid=0 2024-11-10T22:37:45,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073741889_1065 (size=12151) 2024-11-10T22:37:45,563 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:45,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43650 deadline: 1731278325562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:45,564 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:45,564 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:45,564 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:45,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43676 deadline: 1731278325562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:45,564 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:45,565 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:45,565 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:45,565 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:45,573 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:45,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43730 deadline: 1731278325573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:45,574 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:45,574 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:45,574 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:45,574 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:45,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43700 deadline: 1731278325573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:45,574 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:45,574 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:45,574 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:45,574 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:45,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43680 deadline: 1731278325573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:45,575 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:45,575 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:45,575 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:45,658 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/C/c3bc94638ef44755b83a5c137a1b4f2a as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/c3bc94638ef44755b83a5c137a1b4f2a 2024-11-10T22:37:45,668 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in dbbcde27e8d425fcc5932f5841a8c339/C of dbbcde27e8d425fcc5932f5841a8c339 into c3bc94638ef44755b83a5c137a1b4f2a(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T22:37:45,668 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dbbcde27e8d425fcc5932f5841a8c339: 2024-11-10T22:37:45,668 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., storeName=dbbcde27e8d425fcc5932f5841a8c339/C, priority=12, startTime=1731278264754; duration=0sec 2024-11-10T22:37:45,668 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T22:37:45,668 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dbbcde27e8d425fcc5932f5841a8c339:C 2024-11-10T22:37:45,704 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=252 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/B/a9e5790d43df46ac94689c886f408356 2024-11-10T22:37:45,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/C/fa100c3ac50d4b3babf9c1cb9ecca7aa is 50, key is test_row_0/C:col10/1731278264249/Put/seqid=0 2024-11-10T22:37:45,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073741890_1066 (size=12151) 2024-11-10T22:37:45,720 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=252 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/C/fa100c3ac50d4b3babf9c1cb9ecca7aa 2024-11-10T22:37:45,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/A/32d409df8d524c90821cf8dab8e5f762 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/32d409df8d524c90821cf8dab8e5f762 2024-11-10T22:37:45,738 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/32d409df8d524c90821cf8dab8e5f762, entries=150, sequenceid=252, filesize=11.9 K 2024-11-10T22:37:45,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/B/a9e5790d43df46ac94689c886f408356 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/a9e5790d43df46ac94689c886f408356 2024-11-10T22:37:45,747 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/a9e5790d43df46ac94689c886f408356, entries=150, sequenceid=252, filesize=11.9 K 2024-11-10T22:37:45,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/C/fa100c3ac50d4b3babf9c1cb9ecca7aa as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/fa100c3ac50d4b3babf9c1cb9ecca7aa 2024-11-10T22:37:45,760 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/fa100c3ac50d4b3babf9c1cb9ecca7aa, entries=150, sequenceid=252, filesize=11.9 K 2024-11-10T22:37:45,762 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for dbbcde27e8d425fcc5932f5841a8c339 in 898ms, sequenceid=252, compaction requested=false 2024-11-10T22:37:45,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for dbbcde27e8d425fcc5932f5841a8c339: 2024-11-10T22:37:45,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. 2024-11-10T22:37:45,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-11-10T22:37:45,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-11-10T22:37:45,767 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-11-10T22:37:45,767 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6770 sec 2024-11-10T22:37:45,770 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 1.6850 sec 2024-11-10T22:37:46,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(8855): Flush requested on dbbcde27e8d425fcc5932f5841a8c339 2024-11-10T22:37:46,075 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T22:37:46,075 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing dbbcde27e8d425fcc5932f5841a8c339 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-10T22:37:46,083 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/A/26047603692f40d3bd5b6c269bfc0dbc is 50, key is test_row_0/A:col10/1731278264932/Put/seqid=0 2024-11-10T22:37:46,096 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:46,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43730 deadline: 1731278326090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:46,097 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:46,097 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:46,097 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:46,097 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:46,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43650 deadline: 1731278326090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:46,097 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:46,098 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:46,098 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:46,098 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:46,098 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:46,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43700 deadline: 1731278326092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:46,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43676 deadline: 1731278326092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:46,098 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:46,098 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:46,098 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:46,098 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:46,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43680 deadline: 1731278326092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:46,099 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:46,099 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:46,099 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:46,099 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:46,099 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:46,099 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:46,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073741891_1067 (size=14741) 2024-11-10T22:37:46,110 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=279 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/A/26047603692f40d3bd5b6c269bfc0dbc 2024-11-10T22:37:46,121 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/B/5a702cd477014e83a52a63f953db760f is 50, key is test_row_0/B:col10/1731278264932/Put/seqid=0 2024-11-10T22:37:46,133 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073741892_1068 (size=12301) 2024-11-10T22:37:46,207 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:46,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43650 deadline: 1731278326203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:46,208 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:46,208 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:46,208 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:46,208 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:46,208 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:46,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43676 deadline: 1731278326203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:46,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43730 deadline: 1731278326203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:46,209 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:46,208 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:46,209 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:46,209 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:46,209 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:46,209 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:46,209 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:46,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43700 deadline: 1731278326203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:46,209 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:46,209 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:46,209 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:46,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-10T22:37:46,233 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T22:37:46,234 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T22:37:46,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-11-10T22:37:46,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-10T22:37:46,236 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T22:37:46,237 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T22:37:46,237 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T22:37:46,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-10T22:37:46,390 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-10T22:37:46,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. 2024-11-10T22:37:46,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. as already flushing 2024-11-10T22:37:46,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. 2024-11-10T22:37:46,390 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => dbbcde27e8d425fcc5932f5841a8c339, NAME => 'TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:37:46,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => dbbcde27e8d425fcc5932f5841a8c339, NAME => 'TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:37:46,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dbbcde27e8d425fcc5932f5841a8c339, NAME => 'TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dbbcde27e8d425fcc5932f5841a8c339, NAME => 'TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:37:46,415 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:46,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43650 deadline: 1731278326412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:46,416 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:46,416 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:46,416 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:46,416 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:46,416 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:46,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43730 deadline: 1731278326413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:46,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43676 deadline: 1731278326413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:46,416 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:46,416 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:46,417 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:46,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43700 deadline: 1731278326413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:46,417 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:46,417 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:46,417 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:46,417 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:46,417 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:46,417 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:46,417 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:46,534 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=279 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/B/5a702cd477014e83a52a63f953db760f 2024-11-10T22:37:46,544 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-10T22:37:46,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. 2024-11-10T22:37:46,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. as already flushing 2024-11-10T22:37:46,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. 2024-11-10T22:37:46,545 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => dbbcde27e8d425fcc5932f5841a8c339, NAME => 'TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:37:46,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => dbbcde27e8d425fcc5932f5841a8c339, NAME => 'TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:37:46,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dbbcde27e8d425fcc5932f5841a8c339, NAME => 'TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dbbcde27e8d425fcc5932f5841a8c339, NAME => 'TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:37:46,548 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/C/ba4b6037e90c4de184c0f0be2d1df78d is 50, key is test_row_0/C:col10/1731278264932/Put/seqid=0 2024-11-10T22:37:46,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-10T22:37:46,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073741893_1069 (size=12301) 2024-11-10T22:37:46,698 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-10T22:37:46,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. 2024-11-10T22:37:46,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. as already flushing 2024-11-10T22:37:46,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. 2024-11-10T22:37:46,699 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => dbbcde27e8d425fcc5932f5841a8c339, NAME => 'TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:37:46,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => dbbcde27e8d425fcc5932f5841a8c339, NAME => 'TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:37:46,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dbbcde27e8d425fcc5932f5841a8c339, NAME => 'TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dbbcde27e8d425fcc5932f5841a8c339, NAME => 'TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:37:46,723 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:46,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43650 deadline: 1731278326722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:46,724 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:46,724 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:46,724 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:46,724 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:46,724 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:46,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43700 deadline: 1731278326723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:46,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43730 deadline: 1731278326722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:46,725 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:46,725 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:46,725 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:46,725 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:46,725 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:46,725 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:46,725 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:46,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43676 deadline: 1731278326723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:46,725 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:46,725 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:46,725 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:46,853 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-10T22:37:46,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. 2024-11-10T22:37:46,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. as already flushing 2024-11-10T22:37:46,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. 2024-11-10T22:37:46,853 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => dbbcde27e8d425fcc5932f5841a8c339, NAME => 'TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:37:46,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => dbbcde27e8d425fcc5932f5841a8c339, NAME => 'TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:37:46,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dbbcde27e8d425fcc5932f5841a8c339, NAME => 'TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dbbcde27e8d425fcc5932f5841a8c339, NAME => 'TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:37:46,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-10T22:37:46,955 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=279 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/C/ba4b6037e90c4de184c0f0be2d1df78d 2024-11-10T22:37:46,962 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/A/26047603692f40d3bd5b6c269bfc0dbc as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/26047603692f40d3bd5b6c269bfc0dbc 2024-11-10T22:37:46,968 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/26047603692f40d3bd5b6c269bfc0dbc, entries=200, sequenceid=279, filesize=14.4 K 2024-11-10T22:37:46,969 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/B/5a702cd477014e83a52a63f953db760f as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/5a702cd477014e83a52a63f953db760f 2024-11-10T22:37:46,974 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/5a702cd477014e83a52a63f953db760f, entries=150, sequenceid=279, filesize=12.0 K 2024-11-10T22:37:46,976 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/C/ba4b6037e90c4de184c0f0be2d1df78d as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/ba4b6037e90c4de184c0f0be2d1df78d 2024-11-10T22:37:46,982 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/ba4b6037e90c4de184c0f0be2d1df78d, entries=150, sequenceid=279, filesize=12.0 K 2024-11-10T22:37:46,984 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for dbbcde27e8d425fcc5932f5841a8c339 in 908ms, sequenceid=279, compaction requested=true 2024-11-10T22:37:46,984 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for dbbcde27e8d425fcc5932f5841a8c339: 2024-11-10T22:37:46,984 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dbbcde27e8d425fcc5932f5841a8c339:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T22:37:46,984 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T22:37:46,984 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T22:37:46,984 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dbbcde27e8d425fcc5932f5841a8c339:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T22:37:46,984 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T22:37:46,984 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dbbcde27e8d425fcc5932f5841a8c339:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T22:37:46,984 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T22:37:46,984 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T22:37:46,985 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39589 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T22:37:46,985 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HStore(1541): dbbcde27e8d425fcc5932f5841a8c339/A is initiating minor compaction (all files) 2024-11-10T22:37:46,985 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dbbcde27e8d425fcc5932f5841a8c339/A in TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. 2024-11-10T22:37:46,986 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/7ed846158b1e43c3880b1358d6ba8941, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/32d409df8d524c90821cf8dab8e5f762, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/26047603692f40d3bd5b6c269bfc0dbc] into tmpdir=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp, totalSize=38.7 K 2024-11-10T22:37:46,986 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T22:37:46,986 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1541): dbbcde27e8d425fcc5932f5841a8c339/B is initiating minor compaction (all files) 2024-11-10T22:37:46,986 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dbbcde27e8d425fcc5932f5841a8c339/B in TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. 2024-11-10T22:37:46,986 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/f9694985fbdd4980a05aae5d3b026c60, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/a9e5790d43df46ac94689c886f408356, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/5a702cd477014e83a52a63f953db760f] into tmpdir=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp, totalSize=36.3 K 2024-11-10T22:37:46,986 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7ed846158b1e43c3880b1358d6ba8941, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731278263092 2024-11-10T22:37:46,987 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting f9694985fbdd4980a05aae5d3b026c60, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731278263092 2024-11-10T22:37:46,987 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 32d409df8d524c90821cf8dab8e5f762, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1731278264249 2024-11-10T22:37:46,987 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting a9e5790d43df46ac94689c886f408356, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1731278264249 2024-11-10T22:37:46,987 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 26047603692f40d3bd5b6c269bfc0dbc, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1731278264932 2024-11-10T22:37:46,988 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting 5a702cd477014e83a52a63f953db760f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1731278264932 2024-11-10T22:37:47,007 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): dbbcde27e8d425fcc5932f5841a8c339#A#compaction#57 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T22:37:47,007 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-10T22:37:47,007 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): dbbcde27e8d425fcc5932f5841a8c339#B#compaction#58 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T22:37:47,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. 2024-11-10T22:37:47,008 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing dbbcde27e8d425fcc5932f5841a8c339 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-10T22:37:47,008 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/B/f34a46391854427ea79bb8f7a2ffdd6c is 50, key is test_row_0/B:col10/1731278264932/Put/seqid=0 2024-11-10T22:37:47,012 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/A/45c145cc147c4c8e8d16c288e34e57f4 is 50, key is test_row_0/A:col10/1731278264932/Put/seqid=0 2024-11-10T22:37:47,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073741894_1070 (size=12949) 2024-11-10T22:37:47,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/A/de333a7a42014b6c906b6ef354f9f88f is 50, key is test_row_0/A:col10/1731278266088/Put/seqid=0 2024-11-10T22:37:47,026 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/B/f34a46391854427ea79bb8f7a2ffdd6c as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/f34a46391854427ea79bb8f7a2ffdd6c 2024-11-10T22:37:47,039 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dbbcde27e8d425fcc5932f5841a8c339/B of dbbcde27e8d425fcc5932f5841a8c339 into f34a46391854427ea79bb8f7a2ffdd6c(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T22:37:47,039 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dbbcde27e8d425fcc5932f5841a8c339: 2024-11-10T22:37:47,039 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., storeName=dbbcde27e8d425fcc5932f5841a8c339/B, priority=13, startTime=1731278266984; duration=0sec 2024-11-10T22:37:47,039 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T22:37:47,040 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dbbcde27e8d425fcc5932f5841a8c339:B 2024-11-10T22:37:47,040 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T22:37:47,040 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073741895_1071 (size=12949) 2024-11-10T22:37:47,042 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T22:37:47,042 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1541): dbbcde27e8d425fcc5932f5841a8c339/C is initiating minor compaction (all files) 2024-11-10T22:37:47,042 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dbbcde27e8d425fcc5932f5841a8c339/C in TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. 2024-11-10T22:37:47,042 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/c3bc94638ef44755b83a5c137a1b4f2a, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/fa100c3ac50d4b3babf9c1cb9ecca7aa, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/ba4b6037e90c4de184c0f0be2d1df78d] into tmpdir=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp, totalSize=36.3 K 2024-11-10T22:37:47,043 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting c3bc94638ef44755b83a5c137a1b4f2a, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731278263092 2024-11-10T22:37:47,044 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting fa100c3ac50d4b3babf9c1cb9ecca7aa, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1731278264249 2024-11-10T22:37:47,044 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting ba4b6037e90c4de184c0f0be2d1df78d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1731278264932 2024-11-10T22:37:47,051 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/A/45c145cc147c4c8e8d16c288e34e57f4 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/45c145cc147c4c8e8d16c288e34e57f4 2024-11-10T22:37:47,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073741896_1072 (size=12301) 2024-11-10T22:37:47,058 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/A/de333a7a42014b6c906b6ef354f9f88f 2024-11-10T22:37:47,068 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dbbcde27e8d425fcc5932f5841a8c339/A of dbbcde27e8d425fcc5932f5841a8c339 into 45c145cc147c4c8e8d16c288e34e57f4(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T22:37:47,068 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dbbcde27e8d425fcc5932f5841a8c339: 2024-11-10T22:37:47,068 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., storeName=dbbcde27e8d425fcc5932f5841a8c339/A, priority=13, startTime=1731278266984; duration=0sec 2024-11-10T22:37:47,069 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T22:37:47,069 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dbbcde27e8d425fcc5932f5841a8c339:A 2024-11-10T22:37:47,071 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): dbbcde27e8d425fcc5932f5841a8c339#C#compaction#60 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T22:37:47,072 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/C/06ffbce4700542529d0c342773a609f7 is 50, key is test_row_0/C:col10/1731278264932/Put/seqid=0 2024-11-10T22:37:47,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/B/2d73eff4c95e4fa59b29e395d886f941 is 50, key is test_row_0/B:col10/1731278266088/Put/seqid=0 2024-11-10T22:37:47,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073741897_1073 (size=12301) 2024-11-10T22:37:47,080 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/B/2d73eff4c95e4fa59b29e395d886f941 2024-11-10T22:37:47,092 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073741898_1074 (size=12949) 2024-11-10T22:37:47,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/C/7f6b8c49516c43ed8259876beab13e46 is 50, key is test_row_0/C:col10/1731278266088/Put/seqid=0 2024-11-10T22:37:47,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073741899_1075 (size=12301) 2024-11-10T22:37:47,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(8855): Flush requested on dbbcde27e8d425fcc5932f5841a8c339 2024-11-10T22:37:47,107 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. as already flushing 2024-11-10T22:37:47,207 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:47,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43680 deadline: 1731278327203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:47,208 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:47,208 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:47,208 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:47,233 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:47,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43650 deadline: 1731278327232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:47,234 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:47,234 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:47,234 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:47,234 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:47,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43730 deadline: 1731278327233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:47,234 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:47,234 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:47,234 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:47,234 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:47,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43700 deadline: 1731278327233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:47,235 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:47,235 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:47,235 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:47,235 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:47,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43676 deadline: 1731278327233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:47,235 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:47,236 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:47,236 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:47,316 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:47,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43680 deadline: 1731278327313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:47,317 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:47,317 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:47,317 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:47,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-10T22:37:47,501 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/C/7f6b8c49516c43ed8259876beab13e46 2024-11-10T22:37:47,503 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/C/06ffbce4700542529d0c342773a609f7 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/06ffbce4700542529d0c342773a609f7 2024-11-10T22:37:47,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/A/de333a7a42014b6c906b6ef354f9f88f as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/de333a7a42014b6c906b6ef354f9f88f 2024-11-10T22:37:47,513 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dbbcde27e8d425fcc5932f5841a8c339/C of dbbcde27e8d425fcc5932f5841a8c339 into 06ffbce4700542529d0c342773a609f7(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T22:37:47,513 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dbbcde27e8d425fcc5932f5841a8c339: 2024-11-10T22:37:47,513 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., storeName=dbbcde27e8d425fcc5932f5841a8c339/C, priority=13, startTime=1731278266984; duration=0sec 2024-11-10T22:37:47,513 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T22:37:47,513 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dbbcde27e8d425fcc5932f5841a8c339:C 2024-11-10T22:37:47,517 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/de333a7a42014b6c906b6ef354f9f88f, entries=150, sequenceid=291, filesize=12.0 K 2024-11-10T22:37:47,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/B/2d73eff4c95e4fa59b29e395d886f941 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/2d73eff4c95e4fa59b29e395d886f941 2024-11-10T22:37:47,523 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:47,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43680 deadline: 1731278327522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:47,524 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/2d73eff4c95e4fa59b29e395d886f941, entries=150, sequenceid=291, filesize=12.0 K 2024-11-10T22:37:47,526 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:47,526 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:47,526 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:47,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/C/7f6b8c49516c43ed8259876beab13e46 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/7f6b8c49516c43ed8259876beab13e46 2024-11-10T22:37:47,531 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/7f6b8c49516c43ed8259876beab13e46, entries=150, sequenceid=291, filesize=12.0 K 2024-11-10T22:37:47,533 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for dbbcde27e8d425fcc5932f5841a8c339 in 525ms, sequenceid=291, compaction requested=false 2024-11-10T22:37:47,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for dbbcde27e8d425fcc5932f5841a8c339: 2024-11-10T22:37:47,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. 2024-11-10T22:37:47,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-11-10T22:37:47,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-11-10T22:37:47,537 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-10T22:37:47,537 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2980 sec 2024-11-10T22:37:47,540 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 1.3040 sec 2024-11-10T22:37:47,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(8855): Flush requested on dbbcde27e8d425fcc5932f5841a8c339 2024-11-10T22:37:47,836 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T22:37:47,836 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing dbbcde27e8d425fcc5932f5841a8c339 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-10T22:37:47,843 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/A/8c7765598d6c4c6e82a8612db5b51cfd is 50, key is test_row_0/A:col10/1731278267834/Put/seqid=0 2024-11-10T22:37:47,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073741900_1076 (size=14741) 2024-11-10T22:37:47,859 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/A/8c7765598d6c4c6e82a8612db5b51cfd 2024-11-10T22:37:47,874 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/B/bc85c7e6cfd943e79b1463e6784f0663 is 50, key is test_row_0/B:col10/1731278267834/Put/seqid=0 2024-11-10T22:37:47,878 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:47,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43680 deadline: 1731278327876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:47,879 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:47,879 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:47,879 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:47,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073741901_1077 (size=12301) 2024-11-10T22:37:47,983 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:47,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43680 deadline: 1731278327982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:47,984 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:47,984 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:47,984 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:48,195 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:48,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43680 deadline: 1731278328193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:48,196 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:48,196 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:48,196 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:48,243 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:48,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43650 deadline: 1731278328242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:48,244 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:48,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43700 deadline: 1731278328242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:48,245 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:48,244 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 849d0d71310f,43753,1731278250663 for 'test_row_0' in dbbcde27e8d425fcc5932f5841a8c339 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2154 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-10T22:37:48,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43676 deadline: 1731278328242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:48,245 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:48,245 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:48,245 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:48,245 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 849d0d71310f,43753,1731278250663 for 'test_row_0' in dbbcde27e8d425fcc5932f5841a8c339 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2153 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-10T22:37:48,245 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:48,245 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:48,246 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:48,246 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 849d0d71310f,43753,1731278250663 for 'test_row_2' in dbbcde27e8d425fcc5932f5841a8c339 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2154 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-10T22:37:48,246 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:48,246 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:48,247 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:48,256 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:48,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43730 deadline: 1731278328253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:48,256 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 849d0d71310f,43753,1731278250663 for 'test_row_0' in dbbcde27e8d425fcc5932f5841a8c339 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2166 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-10T22:37:48,257 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:48,257 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:48,258 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:48,282 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/B/bc85c7e6cfd943e79b1463e6784f0663 2024-11-10T22:37:48,294 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/C/c150439f212f4834afc76f7dc510ee6b is 50, key is test_row_0/C:col10/1731278267834/Put/seqid=0 2024-11-10T22:37:48,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073741902_1078 (size=12301) 2024-11-10T22:37:48,300 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/C/c150439f212f4834afc76f7dc510ee6b 2024-11-10T22:37:48,307 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/A/8c7765598d6c4c6e82a8612db5b51cfd as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/8c7765598d6c4c6e82a8612db5b51cfd 2024-11-10T22:37:48,313 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/8c7765598d6c4c6e82a8612db5b51cfd, entries=200, sequenceid=319, filesize=14.4 K 2024-11-10T22:37:48,315 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/B/bc85c7e6cfd943e79b1463e6784f0663 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/bc85c7e6cfd943e79b1463e6784f0663 2024-11-10T22:37:48,321 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/bc85c7e6cfd943e79b1463e6784f0663, entries=150, sequenceid=319, filesize=12.0 K 2024-11-10T22:37:48,322 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/C/c150439f212f4834afc76f7dc510ee6b as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/c150439f212f4834afc76f7dc510ee6b 2024-11-10T22:37:48,335 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/c150439f212f4834afc76f7dc510ee6b, entries=150, sequenceid=319, filesize=12.0 K 2024-11-10T22:37:48,336 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for dbbcde27e8d425fcc5932f5841a8c339 in 500ms, sequenceid=319, compaction requested=true 2024-11-10T22:37:48,336 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for dbbcde27e8d425fcc5932f5841a8c339: 2024-11-10T22:37:48,337 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dbbcde27e8d425fcc5932f5841a8c339:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T22:37:48,337 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T22:37:48,337 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T22:37:48,337 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dbbcde27e8d425fcc5932f5841a8c339:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T22:37:48,337 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T22:37:48,337 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dbbcde27e8d425fcc5932f5841a8c339:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T22:37:48,337 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T22:37:48,337 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T22:37:48,339 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39991 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T22:37:48,339 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HStore(1541): dbbcde27e8d425fcc5932f5841a8c339/A is initiating minor compaction (all files) 2024-11-10T22:37:48,340 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dbbcde27e8d425fcc5932f5841a8c339/A in TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. 2024-11-10T22:37:48,340 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/45c145cc147c4c8e8d16c288e34e57f4, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/de333a7a42014b6c906b6ef354f9f88f, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/8c7765598d6c4c6e82a8612db5b51cfd] into tmpdir=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp, totalSize=39.1 K 2024-11-10T22:37:48,340 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T22:37:48,340 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1541): dbbcde27e8d425fcc5932f5841a8c339/B is initiating minor compaction (all files) 2024-11-10T22:37:48,340 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dbbcde27e8d425fcc5932f5841a8c339/B in TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. 2024-11-10T22:37:48,340 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/f34a46391854427ea79bb8f7a2ffdd6c, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/2d73eff4c95e4fa59b29e395d886f941, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/bc85c7e6cfd943e79b1463e6784f0663] into tmpdir=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp, totalSize=36.7 K 2024-11-10T22:37:48,340 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 45c145cc147c4c8e8d16c288e34e57f4, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1731278264932 2024-11-10T22:37:48,341 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting de333a7a42014b6c906b6ef354f9f88f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1731278266088 2024-11-10T22:37:48,341 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8c7765598d6c4c6e82a8612db5b51cfd, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1731278267188 2024-11-10T22:37:48,343 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting f34a46391854427ea79bb8f7a2ffdd6c, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1731278264932 2024-11-10T22:37:48,344 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting 2d73eff4c95e4fa59b29e395d886f941, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1731278266088 2024-11-10T22:37:48,345 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting bc85c7e6cfd943e79b1463e6784f0663, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1731278267188 2024-11-10T22:37:48,366 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): dbbcde27e8d425fcc5932f5841a8c339#A#compaction#66 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T22:37:48,367 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/A/f8b9838d86664305a0c7ba6d38c86560 is 50, key is test_row_0/A:col10/1731278267834/Put/seqid=0 2024-11-10T22:37:48,367 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): dbbcde27e8d425fcc5932f5841a8c339#B#compaction#67 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T22:37:48,368 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/B/947a5f8ac53d450992c6ee072d14bf78 is 50, key is test_row_0/B:col10/1731278267834/Put/seqid=0 2024-11-10T22:37:48,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-10T22:37:48,385 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T22:37:48,385 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T22:37:48,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-11-10T22:37:48,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-10T22:37:48,389 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T22:37:48,391 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T22:37:48,391 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T22:37:48,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073741904_1080 (size=13051) 2024-11-10T22:37:48,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073741903_1079 (size=13051) 2024-11-10T22:37:48,413 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/B/947a5f8ac53d450992c6ee072d14bf78 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/947a5f8ac53d450992c6ee072d14bf78 2024-11-10T22:37:48,424 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dbbcde27e8d425fcc5932f5841a8c339/B of dbbcde27e8d425fcc5932f5841a8c339 into 947a5f8ac53d450992c6ee072d14bf78(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T22:37:48,424 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dbbcde27e8d425fcc5932f5841a8c339: 2024-11-10T22:37:48,424 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., storeName=dbbcde27e8d425fcc5932f5841a8c339/B, priority=13, startTime=1731278268337; duration=0sec 2024-11-10T22:37:48,424 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T22:37:48,424 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dbbcde27e8d425fcc5932f5841a8c339:B 2024-11-10T22:37:48,425 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T22:37:48,426 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T22:37:48,426 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1541): dbbcde27e8d425fcc5932f5841a8c339/C is initiating minor compaction (all files) 2024-11-10T22:37:48,426 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dbbcde27e8d425fcc5932f5841a8c339/C in TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. 2024-11-10T22:37:48,426 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/06ffbce4700542529d0c342773a609f7, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/7f6b8c49516c43ed8259876beab13e46, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/c150439f212f4834afc76f7dc510ee6b] into tmpdir=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp, totalSize=36.7 K 2024-11-10T22:37:48,427 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting 06ffbce4700542529d0c342773a609f7, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1731278264932 2024-11-10T22:37:48,427 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting 7f6b8c49516c43ed8259876beab13e46, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1731278266088 2024-11-10T22:37:48,428 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting c150439f212f4834afc76f7dc510ee6b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1731278267188 2024-11-10T22:37:48,448 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): dbbcde27e8d425fcc5932f5841a8c339#C#compaction#68 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T22:37:48,450 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/C/7c2bc1b4217140ebabd5eb401f1075ea is 50, key is test_row_0/C:col10/1731278267834/Put/seqid=0 2024-11-10T22:37:48,460 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073741905_1081 (size=13051) 2024-11-10T22:37:48,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-10T22:37:48,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(8855): Flush requested on dbbcde27e8d425fcc5932f5841a8c339 2024-11-10T22:37:48,505 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T22:37:48,505 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing dbbcde27e8d425fcc5932f5841a8c339 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-10T22:37:48,511 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/A/2b6c1d275ca44d89ac048b8ec3e154ad is 50, key is test_row_0/A:col10/1731278267857/Put/seqid=0 2024-11-10T22:37:48,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073741906_1082 (size=12297) 2024-11-10T22:37:48,549 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-10T22:37:48,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. 2024-11-10T22:37:48,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. as already flushing 2024-11-10T22:37:48,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. 2024-11-10T22:37:48,550 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => dbbcde27e8d425fcc5932f5841a8c339, NAME => 'TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:37:48,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => dbbcde27e8d425fcc5932f5841a8c339, NAME => 'TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:37:48,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dbbcde27e8d425fcc5932f5841a8c339, NAME => 'TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dbbcde27e8d425fcc5932f5841a8c339, NAME => 'TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:37:48,593 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:48,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43680 deadline: 1731278328589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:48,594 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:48,594 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:48,594 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:48,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-10T22:37:48,703 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-10T22:37:48,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. 2024-11-10T22:37:48,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. as already flushing 2024-11-10T22:37:48,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. 2024-11-10T22:37:48,704 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => dbbcde27e8d425fcc5932f5841a8c339, NAME => 'TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:37:48,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => dbbcde27e8d425fcc5932f5841a8c339, NAME => 'TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:37:48,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dbbcde27e8d425fcc5932f5841a8c339, NAME => 'TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dbbcde27e8d425fcc5932f5841a8c339, NAME => 'TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:37:48,705 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:48,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43680 deadline: 1731278328703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:48,706 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:48,707 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:48,707 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:48,820 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/A/f8b9838d86664305a0c7ba6d38c86560 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/f8b9838d86664305a0c7ba6d38c86560 2024-11-10T22:37:48,829 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dbbcde27e8d425fcc5932f5841a8c339/A of dbbcde27e8d425fcc5932f5841a8c339 into f8b9838d86664305a0c7ba6d38c86560(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T22:37:48,830 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dbbcde27e8d425fcc5932f5841a8c339: 2024-11-10T22:37:48,830 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., storeName=dbbcde27e8d425fcc5932f5841a8c339/A, priority=13, startTime=1731278268336; duration=0sec 2024-11-10T22:37:48,830 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T22:37:48,830 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dbbcde27e8d425fcc5932f5841a8c339:A 2024-11-10T22:37:48,858 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-10T22:37:48,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. 2024-11-10T22:37:48,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. as already flushing 2024-11-10T22:37:48,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. 2024-11-10T22:37:48,858 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => dbbcde27e8d425fcc5932f5841a8c339, NAME => 'TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:37:48,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => dbbcde27e8d425fcc5932f5841a8c339, NAME => 'TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:37:48,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dbbcde27e8d425fcc5932f5841a8c339, NAME => 'TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dbbcde27e8d425fcc5932f5841a8c339, NAME => 'TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:37:48,869 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/C/7c2bc1b4217140ebabd5eb401f1075ea as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/7c2bc1b4217140ebabd5eb401f1075ea 2024-11-10T22:37:48,877 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dbbcde27e8d425fcc5932f5841a8c339/C of dbbcde27e8d425fcc5932f5841a8c339 into 7c2bc1b4217140ebabd5eb401f1075ea(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T22:37:48,878 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dbbcde27e8d425fcc5932f5841a8c339: 2024-11-10T22:37:48,878 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., storeName=dbbcde27e8d425fcc5932f5841a8c339/C, priority=13, startTime=1731278268337; duration=0sec 2024-11-10T22:37:48,878 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T22:37:48,878 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dbbcde27e8d425fcc5932f5841a8c339:C 2024-11-10T22:37:48,913 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:48,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43680 deadline: 1731278328913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:48,914 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:48,914 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:48,914 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:48,935 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=332 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/A/2b6c1d275ca44d89ac048b8ec3e154ad 2024-11-10T22:37:48,947 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/B/1d75f11d9d2d45fd876df00c6139f638 is 50, key is test_row_0/B:col10/1731278267857/Put/seqid=0 2024-11-10T22:37:48,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073741907_1083 (size=9857) 2024-11-10T22:37:49,012 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-10T22:37:49,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. 2024-11-10T22:37:49,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. as already flushing 2024-11-10T22:37:49,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-10T22:37:49,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. 2024-11-10T22:37:49,013 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => dbbcde27e8d425fcc5932f5841a8c339, NAME => 'TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:37:49,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => dbbcde27e8d425fcc5932f5841a8c339, NAME => 'TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:37:49,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dbbcde27e8d425fcc5932f5841a8c339, NAME => 'TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dbbcde27e8d425fcc5932f5841a8c339, NAME => 'TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:37:49,167 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-10T22:37:49,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. 2024-11-10T22:37:49,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. as already flushing 2024-11-10T22:37:49,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. 2024-11-10T22:37:49,168 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => dbbcde27e8d425fcc5932f5841a8c339, NAME => 'TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:37:49,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => dbbcde27e8d425fcc5932f5841a8c339, NAME => 'TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:37:49,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dbbcde27e8d425fcc5932f5841a8c339, NAME => 'TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dbbcde27e8d425fcc5932f5841a8c339, NAME => 'TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:37:49,225 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:49,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43680 deadline: 1731278329223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:49,225 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:49,225 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:49,225 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:49,322 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-10T22:37:49,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. 2024-11-10T22:37:49,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. as already flushing 2024-11-10T22:37:49,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. 2024-11-10T22:37:49,322 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => dbbcde27e8d425fcc5932f5841a8c339, NAME => 'TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:37:49,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => dbbcde27e8d425fcc5932f5841a8c339, NAME => 'TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:37:49,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dbbcde27e8d425fcc5932f5841a8c339, NAME => 'TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dbbcde27e8d425fcc5932f5841a8c339, NAME => 'TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:37:49,371 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=332 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/B/1d75f11d9d2d45fd876df00c6139f638 2024-11-10T22:37:49,382 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/C/dbadc1b2c799480b911a9d5c3c8c5044 is 50, key is test_row_0/C:col10/1731278267857/Put/seqid=0 2024-11-10T22:37:49,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073741908_1084 (size=9857) 2024-11-10T22:37:49,391 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=332 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/C/dbadc1b2c799480b911a9d5c3c8c5044 2024-11-10T22:37:49,400 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/A/2b6c1d275ca44d89ac048b8ec3e154ad as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/2b6c1d275ca44d89ac048b8ec3e154ad 2024-11-10T22:37:49,412 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/2b6c1d275ca44d89ac048b8ec3e154ad, entries=150, sequenceid=332, filesize=12.0 K 2024-11-10T22:37:49,416 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/B/1d75f11d9d2d45fd876df00c6139f638 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/1d75f11d9d2d45fd876df00c6139f638 2024-11-10T22:37:49,429 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/1d75f11d9d2d45fd876df00c6139f638, entries=100, sequenceid=332, filesize=9.6 K 2024-11-10T22:37:49,431 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/C/dbadc1b2c799480b911a9d5c3c8c5044 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/dbadc1b2c799480b911a9d5c3c8c5044 2024-11-10T22:37:49,439 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/dbadc1b2c799480b911a9d5c3c8c5044, entries=100, sequenceid=332, filesize=9.6 K 2024-11-10T22:37:49,440 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for dbbcde27e8d425fcc5932f5841a8c339 in 935ms, sequenceid=332, compaction requested=false 2024-11-10T22:37:49,440 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for dbbcde27e8d425fcc5932f5841a8c339: 2024-11-10T22:37:49,475 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-10T22:37:49,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. 2024-11-10T22:37:49,476 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing dbbcde27e8d425fcc5932f5841a8c339 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-10T22:37:49,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/A/76c2ae53e62a467e9b43a29de62f5918 is 50, key is test_row_0/A:col10/1731278268585/Put/seqid=0 2024-11-10T22:37:49,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073741909_1085 (size=12301) 2024-11-10T22:37:49,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-10T22:37:49,737 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. as already flushing 2024-11-10T22:37:49,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(8855): Flush requested on dbbcde27e8d425fcc5932f5841a8c339 2024-11-10T22:37:49,769 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:49,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43680 deadline: 1731278329766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:49,770 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:49,770 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:49,770 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:49,875 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:49,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43680 deadline: 1731278329872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:49,875 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:49,876 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:49,876 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:49,888 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=358 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/A/76c2ae53e62a467e9b43a29de62f5918 2024-11-10T22:37:49,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/B/618e4af2aa204aebbbddac8ebf481b65 is 50, key is test_row_0/B:col10/1731278268585/Put/seqid=0 2024-11-10T22:37:49,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073741910_1086 (size=12301) 2024-11-10T22:37:49,910 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=358 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/B/618e4af2aa204aebbbddac8ebf481b65 2024-11-10T22:37:49,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/C/02dffc988c654acdb7c5b6636b1f8fb0 is 50, key is test_row_0/C:col10/1731278268585/Put/seqid=0 2024-11-10T22:37:49,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073741911_1087 (size=12301) 2024-11-10T22:37:49,932 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=358 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/C/02dffc988c654acdb7c5b6636b1f8fb0 2024-11-10T22:37:49,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/A/76c2ae53e62a467e9b43a29de62f5918 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/76c2ae53e62a467e9b43a29de62f5918 2024-11-10T22:37:49,950 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/76c2ae53e62a467e9b43a29de62f5918, entries=150, sequenceid=358, filesize=12.0 K 2024-11-10T22:37:49,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/B/618e4af2aa204aebbbddac8ebf481b65 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/618e4af2aa204aebbbddac8ebf481b65 2024-11-10T22:37:49,959 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/618e4af2aa204aebbbddac8ebf481b65, entries=150, sequenceid=358, filesize=12.0 K 2024-11-10T22:37:49,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/C/02dffc988c654acdb7c5b6636b1f8fb0 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/02dffc988c654acdb7c5b6636b1f8fb0 2024-11-10T22:37:49,966 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/02dffc988c654acdb7c5b6636b1f8fb0, entries=150, sequenceid=358, filesize=12.0 K 2024-11-10T22:37:49,968 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for dbbcde27e8d425fcc5932f5841a8c339 in 491ms, sequenceid=358, compaction requested=true 2024-11-10T22:37:49,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for dbbcde27e8d425fcc5932f5841a8c339: 2024-11-10T22:37:49,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. 2024-11-10T22:37:49,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-11-10T22:37:49,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-11-10T22:37:49,975 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-10T22:37:49,975 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5790 sec 2024-11-10T22:37:49,982 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 1.5950 sec 2024-11-10T22:37:50,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(8855): Flush requested on dbbcde27e8d425fcc5932f5841a8c339 2024-11-10T22:37:50,086 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T22:37:50,086 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing dbbcde27e8d425fcc5932f5841a8c339 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-10T22:37:50,092 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/A/9c21f39a2b9b4aa8bd9dbd725423bf4d is 50, key is test_row_0/A:col10/1731278270084/Put/seqid=0 2024-11-10T22:37:50,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073741912_1088 (size=14741) 2024-11-10T22:37:50,190 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:50,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43680 deadline: 1731278330184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:50,191 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:50,191 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:50,191 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:50,253 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:50,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43700 deadline: 1731278330253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:50,254 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:50,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43676 deadline: 1731278330253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:50,254 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 849d0d71310f,43753,1731278250663 for 'test_row_0' in dbbcde27e8d425fcc5932f5841a8c339 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4162 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-10T22:37:50,255 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:50,255 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:50,255 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:50,255 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 849d0d71310f,43753,1731278250663 for 'test_row_2' in dbbcde27e8d425fcc5932f5841a8c339 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4163 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-10T22:37:50,256 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:50,256 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:50,256 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:50,267 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:50,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43650 deadline: 1731278330262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:50,268 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 849d0d71310f,43753,1731278250663 for 'test_row_0' in dbbcde27e8d425fcc5932f5841a8c339 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4178 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-10T22:37:50,268 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:50,268 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:50,268 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:50,274 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:50,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43730 deadline: 1731278330272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:50,274 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 849d0d71310f,43753,1731278250663 for 'test_row_0' in dbbcde27e8d425fcc5932f5841a8c339 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4184 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-10T22:37:50,275 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:50,275 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:50,275 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:50,305 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:50,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43680 deadline: 1731278330302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:50,306 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:50,306 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:50,306 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:50,504 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=371 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/A/9c21f39a2b9b4aa8bd9dbd725423bf4d 2024-11-10T22:37:50,513 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/B/6ea9d03eccff41fbb6a7c1c46f03a547 is 50, key is test_row_0/B:col10/1731278270084/Put/seqid=0 2024-11-10T22:37:50,515 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:50,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43680 deadline: 1731278330512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:50,517 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:50,517 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:50,517 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:50,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073741913_1089 (size=12301) 2024-11-10T22:37:50,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-10T22:37:50,543 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T22:37:50,544 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T22:37:50,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-11-10T22:37:50,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-10T22:37:50,546 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T22:37:50,547 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T22:37:50,547 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T22:37:50,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-10T22:37:50,700 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-10T22:37:50,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. 2024-11-10T22:37:50,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. as already flushing 2024-11-10T22:37:50,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. 2024-11-10T22:37:50,700 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => dbbcde27e8d425fcc5932f5841a8c339, NAME => 'TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:37:50,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => dbbcde27e8d425fcc5932f5841a8c339, NAME => 'TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:37:50,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dbbcde27e8d425fcc5932f5841a8c339, NAME => 'TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dbbcde27e8d425fcc5932f5841a8c339, NAME => 'TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:37:50,824 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:50,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43680 deadline: 1731278330823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:50,825 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:50,825 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:50,825 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:50,854 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-10T22:37:50,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. 2024-11-10T22:37:50,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. as already flushing 2024-11-10T22:37:50,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. 2024-11-10T22:37:50,855 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => dbbcde27e8d425fcc5932f5841a8c339, NAME => 'TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:37:50,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => dbbcde27e8d425fcc5932f5841a8c339, NAME => 'TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:37:50,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dbbcde27e8d425fcc5932f5841a8c339, NAME => 'TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dbbcde27e8d425fcc5932f5841a8c339, NAME => 'TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:37:50,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-10T22:37:50,919 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=371 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/B/6ea9d03eccff41fbb6a7c1c46f03a547 2024-11-10T22:37:50,931 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/C/41a3911cc8004669b1827020c2992317 is 50, key is test_row_0/C:col10/1731278270084/Put/seqid=0 2024-11-10T22:37:50,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073741914_1090 (size=12301) 2024-11-10T22:37:50,944 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=371 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/C/41a3911cc8004669b1827020c2992317 2024-11-10T22:37:50,950 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/A/9c21f39a2b9b4aa8bd9dbd725423bf4d as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/9c21f39a2b9b4aa8bd9dbd725423bf4d 2024-11-10T22:37:50,956 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/9c21f39a2b9b4aa8bd9dbd725423bf4d, entries=200, sequenceid=371, filesize=14.4 K 2024-11-10T22:37:50,957 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/B/6ea9d03eccff41fbb6a7c1c46f03a547 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/6ea9d03eccff41fbb6a7c1c46f03a547 2024-11-10T22:37:50,964 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/6ea9d03eccff41fbb6a7c1c46f03a547, entries=150, sequenceid=371, filesize=12.0 K 2024-11-10T22:37:50,966 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/C/41a3911cc8004669b1827020c2992317 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/41a3911cc8004669b1827020c2992317 2024-11-10T22:37:50,972 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/41a3911cc8004669b1827020c2992317, entries=150, sequenceid=371, filesize=12.0 K 2024-11-10T22:37:50,973 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for dbbcde27e8d425fcc5932f5841a8c339 in 887ms, sequenceid=371, compaction requested=true 2024-11-10T22:37:50,973 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for dbbcde27e8d425fcc5932f5841a8c339: 2024-11-10T22:37:50,973 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dbbcde27e8d425fcc5932f5841a8c339:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T22:37:50,973 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T22:37:50,973 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T22:37:50,973 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dbbcde27e8d425fcc5932f5841a8c339:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T22:37:50,973 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T22:37:50,973 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dbbcde27e8d425fcc5932f5841a8c339:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T22:37:50,973 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T22:37:50,973 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T22:37:50,975 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52390 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T22:37:50,975 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47510 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T22:37:50,975 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HStore(1541): dbbcde27e8d425fcc5932f5841a8c339/A is initiating minor compaction (all files) 2024-11-10T22:37:50,975 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dbbcde27e8d425fcc5932f5841a8c339/A in TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. 2024-11-10T22:37:50,975 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1541): dbbcde27e8d425fcc5932f5841a8c339/B is initiating minor compaction (all files) 2024-11-10T22:37:50,975 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/f8b9838d86664305a0c7ba6d38c86560, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/2b6c1d275ca44d89ac048b8ec3e154ad, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/76c2ae53e62a467e9b43a29de62f5918, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/9c21f39a2b9b4aa8bd9dbd725423bf4d] into tmpdir=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp, totalSize=51.2 K 2024-11-10T22:37:50,975 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dbbcde27e8d425fcc5932f5841a8c339/B in TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. 2024-11-10T22:37:50,975 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/947a5f8ac53d450992c6ee072d14bf78, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/1d75f11d9d2d45fd876df00c6139f638, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/618e4af2aa204aebbbddac8ebf481b65, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/6ea9d03eccff41fbb6a7c1c46f03a547] into tmpdir=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp, totalSize=46.4 K 2024-11-10T22:37:50,975 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting f8b9838d86664305a0c7ba6d38c86560, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1731278267188 2024-11-10T22:37:50,976 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting 947a5f8ac53d450992c6ee072d14bf78, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1731278267188 2024-11-10T22:37:50,976 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2b6c1d275ca44d89ac048b8ec3e154ad, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1731278267857 2024-11-10T22:37:50,976 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting 1d75f11d9d2d45fd876df00c6139f638, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1731278267857 2024-11-10T22:37:50,977 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 76c2ae53e62a467e9b43a29de62f5918, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=358, earliestPutTs=1731278268579 2024-11-10T22:37:50,977 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting 618e4af2aa204aebbbddac8ebf481b65, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=358, earliestPutTs=1731278268579 2024-11-10T22:37:50,977 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9c21f39a2b9b4aa8bd9dbd725423bf4d, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=371, earliestPutTs=1731278269759 2024-11-10T22:37:50,978 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting 6ea9d03eccff41fbb6a7c1c46f03a547, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=371, earliestPutTs=1731278269760 2024-11-10T22:37:50,998 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): dbbcde27e8d425fcc5932f5841a8c339#B#compaction#79 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T22:37:50,998 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): dbbcde27e8d425fcc5932f5841a8c339#A#compaction#78 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T22:37:50,999 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/B/022567373b454e43a2dfb3e0adf279f9 is 50, key is test_row_0/B:col10/1731278270084/Put/seqid=0 2024-11-10T22:37:50,999 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/A/d62a206039c5462190da54367ea25796 is 50, key is test_row_0/A:col10/1731278270084/Put/seqid=0 2024-11-10T22:37:51,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073741915_1091 (size=13187) 2024-11-10T22:37:51,009 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-10T22:37:51,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. 2024-11-10T22:37:51,009 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing dbbcde27e8d425fcc5932f5841a8c339 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-10T22:37:51,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073741916_1092 (size=13187) 2024-11-10T22:37:51,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/A/996ca5c651224ad7af261f58bd38f182 is 50, key is test_row_0/A:col10/1731278270160/Put/seqid=0 2024-11-10T22:37:51,019 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/A/d62a206039c5462190da54367ea25796 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/d62a206039c5462190da54367ea25796 2024-11-10T22:37:51,027 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/B/022567373b454e43a2dfb3e0adf279f9 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/022567373b454e43a2dfb3e0adf279f9 2024-11-10T22:37:51,031 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in dbbcde27e8d425fcc5932f5841a8c339/A of dbbcde27e8d425fcc5932f5841a8c339 into d62a206039c5462190da54367ea25796(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T22:37:51,031 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dbbcde27e8d425fcc5932f5841a8c339: 2024-11-10T22:37:51,031 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., storeName=dbbcde27e8d425fcc5932f5841a8c339/A, priority=12, startTime=1731278270973; duration=0sec 2024-11-10T22:37:51,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073741917_1093 (size=12301) 2024-11-10T22:37:51,034 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T22:37:51,034 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dbbcde27e8d425fcc5932f5841a8c339:A 2024-11-10T22:37:51,034 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=394 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/A/996ca5c651224ad7af261f58bd38f182 2024-11-10T22:37:51,034 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T22:37:51,037 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47510 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T22:37:51,037 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in dbbcde27e8d425fcc5932f5841a8c339/B of dbbcde27e8d425fcc5932f5841a8c339 into 022567373b454e43a2dfb3e0adf279f9(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T22:37:51,037 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dbbcde27e8d425fcc5932f5841a8c339: 2024-11-10T22:37:51,037 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., storeName=dbbcde27e8d425fcc5932f5841a8c339/B, priority=12, startTime=1731278270973; duration=0sec 2024-11-10T22:37:51,038 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T22:37:51,038 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dbbcde27e8d425fcc5932f5841a8c339:B 2024-11-10T22:37:51,038 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HStore(1541): dbbcde27e8d425fcc5932f5841a8c339/C is initiating minor compaction (all files) 2024-11-10T22:37:51,038 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dbbcde27e8d425fcc5932f5841a8c339/C in TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. 2024-11-10T22:37:51,039 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/7c2bc1b4217140ebabd5eb401f1075ea, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/dbadc1b2c799480b911a9d5c3c8c5044, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/02dffc988c654acdb7c5b6636b1f8fb0, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/41a3911cc8004669b1827020c2992317] into tmpdir=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp, totalSize=46.4 K 2024-11-10T22:37:51,040 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7c2bc1b4217140ebabd5eb401f1075ea, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1731278267188 2024-11-10T22:37:51,040 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting dbadc1b2c799480b911a9d5c3c8c5044, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1731278267857 2024-11-10T22:37:51,041 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 02dffc988c654acdb7c5b6636b1f8fb0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=358, earliestPutTs=1731278268579 2024-11-10T22:37:51,041 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 41a3911cc8004669b1827020c2992317, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=371, earliestPutTs=1731278269760 2024-11-10T22:37:51,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/B/82cbd36f92134060a4dfe9bd76e502bd is 50, key is test_row_0/B:col10/1731278270160/Put/seqid=0 2024-11-10T22:37:51,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073741918_1094 (size=12301) 2024-11-10T22:37:51,059 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=394 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/B/82cbd36f92134060a4dfe9bd76e502bd 2024-11-10T22:37:51,068 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): dbbcde27e8d425fcc5932f5841a8c339#C#compaction#82 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T22:37:51,068 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/C/2d8d0d7d82954738a297b75d445597be is 50, key is test_row_0/C:col10/1731278270084/Put/seqid=0 2024-11-10T22:37:51,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/C/b895e2363bde4657b03f95790ca56414 is 50, key is test_row_0/C:col10/1731278270160/Put/seqid=0 2024-11-10T22:37:51,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073741920_1096 (size=12301) 2024-11-10T22:37:51,087 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=394 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/C/b895e2363bde4657b03f95790ca56414 2024-11-10T22:37:51,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073741919_1095 (size=13187) 2024-11-10T22:37:51,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/A/996ca5c651224ad7af261f58bd38f182 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/996ca5c651224ad7af261f58bd38f182 2024-11-10T22:37:51,101 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/C/2d8d0d7d82954738a297b75d445597be as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/2d8d0d7d82954738a297b75d445597be 2024-11-10T22:37:51,108 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/996ca5c651224ad7af261f58bd38f182, entries=150, sequenceid=394, filesize=12.0 K 2024-11-10T22:37:51,108 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in dbbcde27e8d425fcc5932f5841a8c339/C of dbbcde27e8d425fcc5932f5841a8c339 into 2d8d0d7d82954738a297b75d445597be(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T22:37:51,108 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dbbcde27e8d425fcc5932f5841a8c339: 2024-11-10T22:37:51,108 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., storeName=dbbcde27e8d425fcc5932f5841a8c339/C, priority=12, startTime=1731278270973; duration=0sec 2024-11-10T22:37:51,108 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T22:37:51,108 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dbbcde27e8d425fcc5932f5841a8c339:C 2024-11-10T22:37:51,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/B/82cbd36f92134060a4dfe9bd76e502bd as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/82cbd36f92134060a4dfe9bd76e502bd 2024-11-10T22:37:51,115 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/82cbd36f92134060a4dfe9bd76e502bd, entries=150, sequenceid=394, filesize=12.0 K 2024-11-10T22:37:51,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/C/b895e2363bde4657b03f95790ca56414 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/b895e2363bde4657b03f95790ca56414 2024-11-10T22:37:51,125 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/b895e2363bde4657b03f95790ca56414, entries=150, sequenceid=394, filesize=12.0 K 2024-11-10T22:37:51,126 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=0 B/0 for dbbcde27e8d425fcc5932f5841a8c339 in 117ms, sequenceid=394, compaction requested=false 2024-11-10T22:37:51,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for dbbcde27e8d425fcc5932f5841a8c339: 2024-11-10T22:37:51,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. 2024-11-10T22:37:51,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-11-10T22:37:51,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-11-10T22:37:51,131 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-11-10T22:37:51,131 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 581 msec 2024-11-10T22:37:51,135 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 588 msec 2024-11-10T22:37:51,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-10T22:37:51,172 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T22:37:51,173 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T22:37:51,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees 2024-11-10T22:37:51,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-10T22:37:51,176 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T22:37:51,177 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T22:37:51,177 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T22:37:51,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-10T22:37:51,330 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-10T22:37:51,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. 2024-11-10T22:37:51,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2603): Flush status journal for dbbcde27e8d425fcc5932f5841a8c339: 2024-11-10T22:37:51,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. 2024-11-10T22:37:51,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=24 2024-11-10T22:37:51,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.HMaster(4169): Remote procedure done, pid=24 2024-11-10T22:37:51,335 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-10T22:37:51,335 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 155 msec 2024-11-10T22:37:51,338 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees in 163 msec 2024-11-10T22:37:51,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(8855): Flush requested on dbbcde27e8d425fcc5932f5841a8c339 2024-11-10T22:37:51,356 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T22:37:51,356 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing dbbcde27e8d425fcc5932f5841a8c339 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-10T22:37:51,366 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/A/4612de31b24f4f21be666c4370008fb4 is 50, key is test_row_0/A:col10/1731278271355/Put/seqid=0 2024-11-10T22:37:51,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073741921_1097 (size=14741) 2024-11-10T22:37:51,383 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=409 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/A/4612de31b24f4f21be666c4370008fb4 2024-11-10T22:37:51,395 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/B/e84d8ff86b6840419c0a17a80f799531 is 50, key is test_row_0/B:col10/1731278271355/Put/seqid=0 2024-11-10T22:37:51,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073741922_1098 (size=12301) 2024-11-10T22:37:51,415 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=409 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/B/e84d8ff86b6840419c0a17a80f799531 2024-11-10T22:37:51,428 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/C/6ce20a8e7b37485d9f4316f87fe029fb is 50, key is test_row_0/C:col10/1731278271355/Put/seqid=0 2024-11-10T22:37:51,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073741923_1099 (size=12301) 2024-11-10T22:37:51,482 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:51,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 218 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43680 deadline: 1731278331479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:51,483 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:51,483 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:51,483 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:51,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-10T22:37:51,493 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T22:37:51,494 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T22:37:51,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees 2024-11-10T22:37:51,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-10T22:37:51,496 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T22:37:51,497 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T22:37:51,497 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T22:37:51,593 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:51,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 219 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43680 deadline: 1731278331593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:51,594 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:51,594 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:51,594 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:51,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-10T22:37:51,649 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-10T22:37:51,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. 2024-11-10T22:37:51,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. as already flushing 2024-11-10T22:37:51,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. 2024-11-10T22:37:51,650 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => dbbcde27e8d425fcc5932f5841a8c339, NAME => 'TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:37:51,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => dbbcde27e8d425fcc5932f5841a8c339, NAME => 'TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:37:51,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dbbcde27e8d425fcc5932f5841a8c339, NAME => 'TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dbbcde27e8d425fcc5932f5841a8c339, NAME => 'TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:37:51,803 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-10T22:37:51,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. 2024-11-10T22:37:51,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. as already flushing 2024-11-10T22:37:51,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. 2024-11-10T22:37:51,804 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => dbbcde27e8d425fcc5932f5841a8c339, NAME => 'TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:37:51,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => dbbcde27e8d425fcc5932f5841a8c339, NAME => 'TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:37:51,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dbbcde27e8d425fcc5932f5841a8c339, NAME => 'TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dbbcde27e8d425fcc5932f5841a8c339, NAME => 'TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:37:51,809 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:51,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 220 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43680 deadline: 1731278331802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:51,810 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:51,810 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:51,810 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:51,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-10T22:37:51,851 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=409 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/C/6ce20a8e7b37485d9f4316f87fe029fb 2024-11-10T22:37:51,858 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/A/4612de31b24f4f21be666c4370008fb4 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/4612de31b24f4f21be666c4370008fb4 2024-11-10T22:37:51,865 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/4612de31b24f4f21be666c4370008fb4, entries=200, sequenceid=409, filesize=14.4 K 2024-11-10T22:37:51,867 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/B/e84d8ff86b6840419c0a17a80f799531 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/e84d8ff86b6840419c0a17a80f799531 2024-11-10T22:37:51,873 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/e84d8ff86b6840419c0a17a80f799531, entries=150, sequenceid=409, filesize=12.0 K 2024-11-10T22:37:51,874 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/C/6ce20a8e7b37485d9f4316f87fe029fb as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/6ce20a8e7b37485d9f4316f87fe029fb 2024-11-10T22:37:51,882 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/6ce20a8e7b37485d9f4316f87fe029fb, entries=150, sequenceid=409, filesize=12.0 K 2024-11-10T22:37:51,883 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for dbbcde27e8d425fcc5932f5841a8c339 in 527ms, sequenceid=409, compaction requested=true 2024-11-10T22:37:51,883 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for dbbcde27e8d425fcc5932f5841a8c339: 2024-11-10T22:37:51,884 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dbbcde27e8d425fcc5932f5841a8c339:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T22:37:51,884 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T22:37:51,884 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T22:37:51,884 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dbbcde27e8d425fcc5932f5841a8c339:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T22:37:51,884 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T22:37:51,884 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T22:37:51,884 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dbbcde27e8d425fcc5932f5841a8c339:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T22:37:51,884 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T22:37:51,887 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40229 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T22:37:51,887 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HStore(1541): dbbcde27e8d425fcc5932f5841a8c339/A is initiating minor compaction (all files) 2024-11-10T22:37:51,888 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dbbcde27e8d425fcc5932f5841a8c339/A in TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. 2024-11-10T22:37:51,888 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/d62a206039c5462190da54367ea25796, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/996ca5c651224ad7af261f58bd38f182, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/4612de31b24f4f21be666c4370008fb4] into tmpdir=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp, totalSize=39.3 K 2024-11-10T22:37:51,888 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T22:37:51,888 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1541): dbbcde27e8d425fcc5932f5841a8c339/B is initiating minor compaction (all files) 2024-11-10T22:37:51,888 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dbbcde27e8d425fcc5932f5841a8c339/B in TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. 2024-11-10T22:37:51,889 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/022567373b454e43a2dfb3e0adf279f9, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/82cbd36f92134060a4dfe9bd76e502bd, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/e84d8ff86b6840419c0a17a80f799531] into tmpdir=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp, totalSize=36.9 K 2024-11-10T22:37:51,889 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting d62a206039c5462190da54367ea25796, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=371, earliestPutTs=1731278269760 2024-11-10T22:37:51,889 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting 022567373b454e43a2dfb3e0adf279f9, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=371, earliestPutTs=1731278269760 2024-11-10T22:37:51,889 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 996ca5c651224ad7af261f58bd38f182, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=394, earliestPutTs=1731278270160 2024-11-10T22:37:51,890 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting 82cbd36f92134060a4dfe9bd76e502bd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=394, earliestPutTs=1731278270160 2024-11-10T22:37:51,890 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting e84d8ff86b6840419c0a17a80f799531, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=409, earliestPutTs=1731278271336 2024-11-10T22:37:51,891 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4612de31b24f4f21be666c4370008fb4, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=409, earliestPutTs=1731278271336 2024-11-10T22:37:51,904 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): dbbcde27e8d425fcc5932f5841a8c339#A#compaction#87 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T22:37:51,904 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/A/ce40513cc5d04cf68bb715a035dcf91a is 50, key is test_row_0/A:col10/1731278271355/Put/seqid=0 2024-11-10T22:37:51,906 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): dbbcde27e8d425fcc5932f5841a8c339#B#compaction#88 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T22:37:51,906 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/B/fef6efd9f5e044c082ad868c095c4a51 is 50, key is test_row_0/B:col10/1731278271355/Put/seqid=0 2024-11-10T22:37:51,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073741924_1100 (size=13289) 2024-11-10T22:37:51,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073741925_1101 (size=13289) 2024-11-10T22:37:51,929 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/A/ce40513cc5d04cf68bb715a035dcf91a as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/ce40513cc5d04cf68bb715a035dcf91a 2024-11-10T22:37:51,936 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dbbcde27e8d425fcc5932f5841a8c339/A of dbbcde27e8d425fcc5932f5841a8c339 into ce40513cc5d04cf68bb715a035dcf91a(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T22:37:51,936 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dbbcde27e8d425fcc5932f5841a8c339: 2024-11-10T22:37:51,936 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., storeName=dbbcde27e8d425fcc5932f5841a8c339/A, priority=13, startTime=1731278271884; duration=0sec 2024-11-10T22:37:51,937 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T22:37:51,937 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dbbcde27e8d425fcc5932f5841a8c339:A 2024-11-10T22:37:51,937 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T22:37:51,940 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T22:37:51,940 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HStore(1541): dbbcde27e8d425fcc5932f5841a8c339/C is initiating minor compaction (all files) 2024-11-10T22:37:51,940 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dbbcde27e8d425fcc5932f5841a8c339/C in TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. 2024-11-10T22:37:51,941 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/2d8d0d7d82954738a297b75d445597be, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/b895e2363bde4657b03f95790ca56414, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/6ce20a8e7b37485d9f4316f87fe029fb] into tmpdir=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp, totalSize=36.9 K 2024-11-10T22:37:51,941 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2d8d0d7d82954738a297b75d445597be, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=371, earliestPutTs=1731278269760 2024-11-10T22:37:51,942 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting b895e2363bde4657b03f95790ca56414, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=394, earliestPutTs=1731278270160 2024-11-10T22:37:51,943 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6ce20a8e7b37485d9f4316f87fe029fb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=409, earliestPutTs=1731278271336 2024-11-10T22:37:51,955 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): dbbcde27e8d425fcc5932f5841a8c339#C#compaction#89 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T22:37:51,955 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/C/3ccf43dd083747d5b1a8f0ff6607ff79 is 50, key is test_row_0/C:col10/1731278271355/Put/seqid=0 2024-11-10T22:37:51,957 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-10T22:37:51,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. 2024-11-10T22:37:51,958 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2902): Flushing dbbcde27e8d425fcc5932f5841a8c339 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-10T22:37:51,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/A/d9e1225883b54667b5752aeb9c01cd68 is 50, key is test_row_0/A:col10/1731278271471/Put/seqid=0 2024-11-10T22:37:51,977 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073741926_1102 (size=13289) 2024-11-10T22:37:51,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073741927_1103 (size=12301) 2024-11-10T22:37:51,983 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=435 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/A/d9e1225883b54667b5752aeb9c01cd68 2024-11-10T22:37:51,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/B/9739cb02b5094287abaa031d46627bed is 50, key is test_row_0/B:col10/1731278271471/Put/seqid=0 2024-11-10T22:37:52,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073741928_1104 (size=12301) 2024-11-10T22:37:52,010 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=435 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/B/9739cb02b5094287abaa031d46627bed 2024-11-10T22:37:52,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/C/57f2f4f1eba44484a47acaf13112c371 is 50, key is test_row_0/C:col10/1731278271471/Put/seqid=0 2024-11-10T22:37:52,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073741929_1105 (size=12301) 2024-11-10T22:37:52,058 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=435 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/C/57f2f4f1eba44484a47acaf13112c371 2024-11-10T22:37:52,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/A/d9e1225883b54667b5752aeb9c01cd68 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/d9e1225883b54667b5752aeb9c01cd68 2024-11-10T22:37:52,077 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/d9e1225883b54667b5752aeb9c01cd68, entries=150, sequenceid=435, filesize=12.0 K 2024-11-10T22:37:52,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/B/9739cb02b5094287abaa031d46627bed as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/9739cb02b5094287abaa031d46627bed 2024-11-10T22:37:52,090 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/9739cb02b5094287abaa031d46627bed, entries=150, sequenceid=435, filesize=12.0 K 2024-11-10T22:37:52,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/C/57f2f4f1eba44484a47acaf13112c371 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/57f2f4f1eba44484a47acaf13112c371 2024-11-10T22:37:52,103 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/57f2f4f1eba44484a47acaf13112c371, entries=150, sequenceid=435, filesize=12.0 K 2024-11-10T22:37:52,104 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=0 B/0 for dbbcde27e8d425fcc5932f5841a8c339 in 146ms, sequenceid=435, compaction requested=false 2024-11-10T22:37:52,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2603): Flush status journal for dbbcde27e8d425fcc5932f5841a8c339: 2024-11-10T22:37:52,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. 2024-11-10T22:37:52,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=26 2024-11-10T22:37:52,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.HMaster(4169): Remote procedure done, pid=26 2024-11-10T22:37:52,108 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-10T22:37:52,108 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 609 msec 2024-11-10T22:37:52,111 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees in 615 msec 2024-11-10T22:37:52,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-10T22:37:52,123 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T22:37:52,123 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T22:37:52,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] procedure2.ProcedureExecutor(1139): Stored pid=27, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees 2024-11-10T22:37:52,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-10T22:37:52,126 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=27, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T22:37:52,127 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=27, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T22:37:52,127 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T22:37:52,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(8855): Flush requested on dbbcde27e8d425fcc5932f5841a8c339 2024-11-10T22:37:52,136 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T22:37:52,136 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing dbbcde27e8d425fcc5932f5841a8c339 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-10T22:37:52,141 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/A/e20c26f86cd14d2e88445aad7199e94b is 50, key is test_row_0/A:col10/1731278272130/Put/seqid=0 2024-11-10T22:37:52,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073741930_1106 (size=14741) 2024-11-10T22:37:52,146 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=446 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/A/e20c26f86cd14d2e88445aad7199e94b 2024-11-10T22:37:52,158 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/B/77c78595567b4faa8ee7b409917fa3d3 is 50, key is test_row_0/B:col10/1731278272130/Put/seqid=0 2024-11-10T22:37:52,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073741931_1107 (size=12301) 2024-11-10T22:37:52,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-10T22:37:52,278 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-10T22:37:52,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. 2024-11-10T22:37:52,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. as already flushing 2024-11-10T22:37:52,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. 2024-11-10T22:37:52,279 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => dbbcde27e8d425fcc5932f5841a8c339, NAME => 'TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:37:52,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => dbbcde27e8d425fcc5932f5841a8c339, NAME => 'TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:37:52,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dbbcde27e8d425fcc5932f5841a8c339, NAME => 'TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dbbcde27e8d425fcc5932f5841a8c339, NAME => 'TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:37:52,281 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:52,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 251 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43680 deadline: 1731278332276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:52,282 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:52,282 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:52,282 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:52,330 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/B/fef6efd9f5e044c082ad868c095c4a51 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/fef6efd9f5e044c082ad868c095c4a51 2024-11-10T22:37:52,338 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dbbcde27e8d425fcc5932f5841a8c339/B of dbbcde27e8d425fcc5932f5841a8c339 into fef6efd9f5e044c082ad868c095c4a51(size=13.0 K), total size for store is 25.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T22:37:52,339 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dbbcde27e8d425fcc5932f5841a8c339: 2024-11-10T22:37:52,339 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., storeName=dbbcde27e8d425fcc5932f5841a8c339/B, priority=13, startTime=1731278271884; duration=0sec 2024-11-10T22:37:52,339 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T22:37:52,339 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dbbcde27e8d425fcc5932f5841a8c339:B 2024-11-10T22:37:52,384 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/C/3ccf43dd083747d5b1a8f0ff6607ff79 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/3ccf43dd083747d5b1a8f0ff6607ff79 2024-11-10T22:37:52,392 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dbbcde27e8d425fcc5932f5841a8c339/C of dbbcde27e8d425fcc5932f5841a8c339 into 3ccf43dd083747d5b1a8f0ff6607ff79(size=13.0 K), total size for store is 25.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T22:37:52,392 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dbbcde27e8d425fcc5932f5841a8c339: 2024-11-10T22:37:52,392 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., storeName=dbbcde27e8d425fcc5932f5841a8c339/C, priority=13, startTime=1731278271884; duration=0sec 2024-11-10T22:37:52,392 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T22:37:52,392 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dbbcde27e8d425fcc5932f5841a8c339:C 2024-11-10T22:37:52,399 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:52,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 252 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43680 deadline: 1731278332392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:52,399 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:52,399 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:52,399 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:52,433 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-10T22:37:52,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. 2024-11-10T22:37:52,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. as already flushing 2024-11-10T22:37:52,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. 2024-11-10T22:37:52,434 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => dbbcde27e8d425fcc5932f5841a8c339, NAME => 'TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:37:52,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => dbbcde27e8d425fcc5932f5841a8c339, NAME => 'TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:37:52,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dbbcde27e8d425fcc5932f5841a8c339, NAME => 'TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dbbcde27e8d425fcc5932f5841a8c339, NAME => 'TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:37:52,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-10T22:37:52,573 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=446 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/B/77c78595567b4faa8ee7b409917fa3d3 2024-11-10T22:37:52,583 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/C/f533c10058cf43c1996bdbfba5f17afe is 50, key is test_row_0/C:col10/1731278272130/Put/seqid=0 2024-11-10T22:37:52,587 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-10T22:37:52,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. 2024-11-10T22:37:52,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. as already flushing 2024-11-10T22:37:52,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. 2024-11-10T22:37:52,587 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => dbbcde27e8d425fcc5932f5841a8c339, NAME => 'TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:37:52,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => dbbcde27e8d425fcc5932f5841a8c339, NAME => 'TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:37:52,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dbbcde27e8d425fcc5932f5841a8c339, NAME => 'TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dbbcde27e8d425fcc5932f5841a8c339, NAME => 'TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:37:52,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073741932_1108 (size=12301) 2024-11-10T22:37:52,592 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=446 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/C/f533c10058cf43c1996bdbfba5f17afe 2024-11-10T22:37:52,599 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/A/e20c26f86cd14d2e88445aad7199e94b as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/e20c26f86cd14d2e88445aad7199e94b 2024-11-10T22:37:52,603 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:52,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 253 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43680 deadline: 1731278332603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:52,604 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:52,604 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:52,604 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:52,605 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/e20c26f86cd14d2e88445aad7199e94b, entries=200, sequenceid=446, filesize=14.4 K 2024-11-10T22:37:52,606 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/B/77c78595567b4faa8ee7b409917fa3d3 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/77c78595567b4faa8ee7b409917fa3d3 2024-11-10T22:37:52,612 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/77c78595567b4faa8ee7b409917fa3d3, entries=150, sequenceid=446, filesize=12.0 K 2024-11-10T22:37:52,615 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/C/f533c10058cf43c1996bdbfba5f17afe as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/f533c10058cf43c1996bdbfba5f17afe 2024-11-10T22:37:52,621 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/f533c10058cf43c1996bdbfba5f17afe, entries=150, sequenceid=446, filesize=12.0 K 2024-11-10T22:37:52,622 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for dbbcde27e8d425fcc5932f5841a8c339 in 486ms, sequenceid=446, compaction requested=true 2024-11-10T22:37:52,622 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for dbbcde27e8d425fcc5932f5841a8c339: 2024-11-10T22:37:52,623 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T22:37:52,623 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dbbcde27e8d425fcc5932f5841a8c339:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T22:37:52,623 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T22:37:52,623 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T22:37:52,623 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dbbcde27e8d425fcc5932f5841a8c339:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T22:37:52,623 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T22:37:52,623 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dbbcde27e8d425fcc5932f5841a8c339:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T22:37:52,623 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T22:37:52,625 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40331 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T22:37:52,625 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HStore(1541): dbbcde27e8d425fcc5932f5841a8c339/A is initiating minor compaction (all files) 2024-11-10T22:37:52,625 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dbbcde27e8d425fcc5932f5841a8c339/A in TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. 2024-11-10T22:37:52,625 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/ce40513cc5d04cf68bb715a035dcf91a, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/d9e1225883b54667b5752aeb9c01cd68, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/e20c26f86cd14d2e88445aad7199e94b] into tmpdir=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp, totalSize=39.4 K 2024-11-10T22:37:52,626 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37891 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T22:37:52,626 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1541): dbbcde27e8d425fcc5932f5841a8c339/B is initiating minor compaction (all files) 2024-11-10T22:37:52,626 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dbbcde27e8d425fcc5932f5841a8c339/B in TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. 2024-11-10T22:37:52,626 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/fef6efd9f5e044c082ad868c095c4a51, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/9739cb02b5094287abaa031d46627bed, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/77c78595567b4faa8ee7b409917fa3d3] into tmpdir=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp, totalSize=37.0 K 2024-11-10T22:37:52,626 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting ce40513cc5d04cf68bb715a035dcf91a, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=409, earliestPutTs=1731278271336 2024-11-10T22:37:52,626 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting fef6efd9f5e044c082ad868c095c4a51, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=409, earliestPutTs=1731278271336 2024-11-10T22:37:52,627 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting d9e1225883b54667b5752aeb9c01cd68, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=435, earliestPutTs=1731278271466 2024-11-10T22:37:52,627 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting e20c26f86cd14d2e88445aad7199e94b, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=446, earliestPutTs=1731278272121 2024-11-10T22:37:52,627 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting 9739cb02b5094287abaa031d46627bed, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=435, earliestPutTs=1731278271466 2024-11-10T22:37:52,628 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting 77c78595567b4faa8ee7b409917fa3d3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=446, earliestPutTs=1731278272127 2024-11-10T22:37:52,641 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): dbbcde27e8d425fcc5932f5841a8c339#B#compaction#96 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T22:37:52,642 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/B/db6eccb0b2e54b92b191435288b0b2bb is 50, key is test_row_0/B:col10/1731278272130/Put/seqid=0 2024-11-10T22:37:52,645 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): dbbcde27e8d425fcc5932f5841a8c339#A#compaction#97 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T22:37:52,646 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/A/3fc52931291045539a3d338411c181bf is 50, key is test_row_0/A:col10/1731278272130/Put/seqid=0 2024-11-10T22:37:52,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073741933_1109 (size=13391) 2024-11-10T22:37:52,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073741934_1110 (size=13391) 2024-11-10T22:37:52,695 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/A/3fc52931291045539a3d338411c181bf as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/3fc52931291045539a3d338411c181bf 2024-11-10T22:37:52,702 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dbbcde27e8d425fcc5932f5841a8c339/A of dbbcde27e8d425fcc5932f5841a8c339 into 3fc52931291045539a3d338411c181bf(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T22:37:52,702 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dbbcde27e8d425fcc5932f5841a8c339: 2024-11-10T22:37:52,702 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., storeName=dbbcde27e8d425fcc5932f5841a8c339/A, priority=13, startTime=1731278272622; duration=0sec 2024-11-10T22:37:52,702 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T22:37:52,702 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dbbcde27e8d425fcc5932f5841a8c339:A 2024-11-10T22:37:52,702 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T22:37:52,703 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37891 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T22:37:52,703 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HStore(1541): dbbcde27e8d425fcc5932f5841a8c339/C is initiating minor compaction (all files) 2024-11-10T22:37:52,703 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dbbcde27e8d425fcc5932f5841a8c339/C in TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. 2024-11-10T22:37:52,703 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/3ccf43dd083747d5b1a8f0ff6607ff79, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/57f2f4f1eba44484a47acaf13112c371, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/f533c10058cf43c1996bdbfba5f17afe] into tmpdir=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp, totalSize=37.0 K 2024-11-10T22:37:52,704 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3ccf43dd083747d5b1a8f0ff6607ff79, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=409, earliestPutTs=1731278271336 2024-11-10T22:37:52,705 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 57f2f4f1eba44484a47acaf13112c371, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=435, earliestPutTs=1731278271466 2024-11-10T22:37:52,705 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting f533c10058cf43c1996bdbfba5f17afe, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=446, earliestPutTs=1731278272127 2024-11-10T22:37:52,716 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): dbbcde27e8d425fcc5932f5841a8c339#C#compaction#98 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T22:37:52,717 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/C/2c424e9f1614487fbebe8b57481cdb54 is 50, key is test_row_0/C:col10/1731278272130/Put/seqid=0 2024-11-10T22:37:52,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073741935_1111 (size=13391) 2024-11-10T22:37:52,740 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-10T22:37:52,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. 2024-11-10T22:37:52,741 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2902): Flushing dbbcde27e8d425fcc5932f5841a8c339 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-10T22:37:52,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/A/9c611724e8ce43109bce3190aba006b3 is 50, key is test_row_0/A:col10/1731278272259/Put/seqid=0 2024-11-10T22:37:52,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073741936_1112 (size=12301) 2024-11-10T22:37:52,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-10T22:37:52,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(8855): Flush requested on dbbcde27e8d425fcc5932f5841a8c339 2024-11-10T22:37:52,917 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. as already flushing 2024-11-10T22:37:52,949 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:52,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 262 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43680 deadline: 1731278332947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:52,950 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:52,950 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:52,950 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:53,055 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:53,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 263 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43680 deadline: 1731278333053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:53,056 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:53,056 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:53,056 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:53,070 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/B/db6eccb0b2e54b92b191435288b0b2bb as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/db6eccb0b2e54b92b191435288b0b2bb 2024-11-10T22:37:53,076 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dbbcde27e8d425fcc5932f5841a8c339/B of dbbcde27e8d425fcc5932f5841a8c339 into db6eccb0b2e54b92b191435288b0b2bb(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T22:37:53,076 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dbbcde27e8d425fcc5932f5841a8c339: 2024-11-10T22:37:53,077 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., storeName=dbbcde27e8d425fcc5932f5841a8c339/B, priority=13, startTime=1731278272623; duration=0sec 2024-11-10T22:37:53,077 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T22:37:53,077 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dbbcde27e8d425fcc5932f5841a8c339:B 2024-11-10T22:37:53,133 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/C/2c424e9f1614487fbebe8b57481cdb54 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/2c424e9f1614487fbebe8b57481cdb54 2024-11-10T22:37:53,140 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dbbcde27e8d425fcc5932f5841a8c339/C of dbbcde27e8d425fcc5932f5841a8c339 into 2c424e9f1614487fbebe8b57481cdb54(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T22:37:53,140 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dbbcde27e8d425fcc5932f5841a8c339: 2024-11-10T22:37:53,140 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., storeName=dbbcde27e8d425fcc5932f5841a8c339/C, priority=13, startTime=1731278272623; duration=0sec 2024-11-10T22:37:53,140 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T22:37:53,140 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dbbcde27e8d425fcc5932f5841a8c339:C 2024-11-10T22:37:53,152 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=474 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/A/9c611724e8ce43109bce3190aba006b3 2024-11-10T22:37:53,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/B/d583253c5f7245fb88c05fadb1974716 is 50, key is test_row_0/B:col10/1731278272259/Put/seqid=0 2024-11-10T22:37:53,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073741937_1113 (size=12301) 2024-11-10T22:37:53,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-10T22:37:53,265 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:53,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 264 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43680 deadline: 1731278333262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:53,266 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:53,266 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:53,266 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:53,575 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:53,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 265 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43680 deadline: 1731278333573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:53,576 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:53,576 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:53,576 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:53,585 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=474 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/B/d583253c5f7245fb88c05fadb1974716 2024-11-10T22:37:53,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/C/6c10a4657cc14291b13fe1750b2dab2f is 50, key is test_row_0/C:col10/1731278272259/Put/seqid=0 2024-11-10T22:37:53,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073741938_1114 (size=12301) 2024-11-10T22:37:54,014 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=474 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/C/6c10a4657cc14291b13fe1750b2dab2f 2024-11-10T22:37:54,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/A/9c611724e8ce43109bce3190aba006b3 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/9c611724e8ce43109bce3190aba006b3 2024-11-10T22:37:54,026 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/9c611724e8ce43109bce3190aba006b3, entries=150, sequenceid=474, filesize=12.0 K 2024-11-10T22:37:54,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/B/d583253c5f7245fb88c05fadb1974716 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/d583253c5f7245fb88c05fadb1974716 2024-11-10T22:37:54,033 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/d583253c5f7245fb88c05fadb1974716, entries=150, sequenceid=474, filesize=12.0 K 2024-11-10T22:37:54,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/C/6c10a4657cc14291b13fe1750b2dab2f as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/6c10a4657cc14291b13fe1750b2dab2f 2024-11-10T22:37:54,039 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/6c10a4657cc14291b13fe1750b2dab2f, entries=150, sequenceid=474, filesize=12.0 K 2024-11-10T22:37:54,040 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for dbbcde27e8d425fcc5932f5841a8c339 in 1298ms, sequenceid=474, compaction requested=false 2024-11-10T22:37:54,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2603): Flush status journal for dbbcde27e8d425fcc5932f5841a8c339: 2024-11-10T22:37:54,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. 2024-11-10T22:37:54,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=28 2024-11-10T22:37:54,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.HMaster(4169): Remote procedure done, pid=28 2024-11-10T22:37:54,044 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-11-10T22:37:54,044 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9140 sec 2024-11-10T22:37:54,046 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees in 1.9200 sec 2024-11-10T22:37:54,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(8855): Flush requested on dbbcde27e8d425fcc5932f5841a8c339 2024-11-10T22:37:54,084 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T22:37:54,084 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing dbbcde27e8d425fcc5932f5841a8c339 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-10T22:37:54,090 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/A/24be0ad3d9c64efb9e4053b80350110c is 50, key is test_row_0/A:col10/1731278272945/Put/seqid=0 2024-11-10T22:37:54,095 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073741939_1115 (size=12297) 2024-11-10T22:37:54,096 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=488 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/A/24be0ad3d9c64efb9e4053b80350110c 2024-11-10T22:37:54,106 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/B/ad11d27f247c437f83e6bc6dae42b5fb is 50, key is test_row_0/B:col10/1731278272945/Put/seqid=0 2024-11-10T22:37:54,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073741940_1116 (size=9857) 2024-11-10T22:37:54,123 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=488 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/B/ad11d27f247c437f83e6bc6dae42b5fb 2024-11-10T22:37:54,136 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/C/06eed5fa99724f70af604cb7a95d7903 is 50, key is test_row_0/C:col10/1731278272945/Put/seqid=0 2024-11-10T22:37:54,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073741941_1117 (size=9857) 2024-11-10T22:37:54,146 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=488 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/C/06eed5fa99724f70af604cb7a95d7903 2024-11-10T22:37:54,153 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/A/24be0ad3d9c64efb9e4053b80350110c as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/24be0ad3d9c64efb9e4053b80350110c 2024-11-10T22:37:54,159 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/24be0ad3d9c64efb9e4053b80350110c, entries=150, sequenceid=488, filesize=12.0 K 2024-11-10T22:37:54,160 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/B/ad11d27f247c437f83e6bc6dae42b5fb as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/ad11d27f247c437f83e6bc6dae42b5fb 2024-11-10T22:37:54,167 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/ad11d27f247c437f83e6bc6dae42b5fb, entries=100, sequenceid=488, filesize=9.6 K 2024-11-10T22:37:54,168 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/C/06eed5fa99724f70af604cb7a95d7903 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/06eed5fa99724f70af604cb7a95d7903 2024-11-10T22:37:54,169 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:54,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 288 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43680 deadline: 1731278334166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:54,170 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:54,170 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:54,170 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:54,173 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/06eed5fa99724f70af604cb7a95d7903, entries=100, sequenceid=488, filesize=9.6 K 2024-11-10T22:37:54,174 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for dbbcde27e8d425fcc5932f5841a8c339 in 90ms, sequenceid=488, compaction requested=true 2024-11-10T22:37:54,174 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for dbbcde27e8d425fcc5932f5841a8c339: 2024-11-10T22:37:54,174 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dbbcde27e8d425fcc5932f5841a8c339:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T22:37:54,174 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T22:37:54,174 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T22:37:54,174 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dbbcde27e8d425fcc5932f5841a8c339:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T22:37:54,174 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T22:37:54,174 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dbbcde27e8d425fcc5932f5841a8c339:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T22:37:54,174 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T22:37:54,174 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T22:37:54,175 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35549 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T22:37:54,175 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1541): dbbcde27e8d425fcc5932f5841a8c339/B is initiating minor compaction (all files) 2024-11-10T22:37:54,175 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dbbcde27e8d425fcc5932f5841a8c339/B in TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. 2024-11-10T22:37:54,176 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/db6eccb0b2e54b92b191435288b0b2bb, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/d583253c5f7245fb88c05fadb1974716, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/ad11d27f247c437f83e6bc6dae42b5fb] into tmpdir=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp, totalSize=34.7 K 2024-11-10T22:37:54,176 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37989 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T22:37:54,176 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HStore(1541): dbbcde27e8d425fcc5932f5841a8c339/A is initiating minor compaction (all files) 2024-11-10T22:37:54,176 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dbbcde27e8d425fcc5932f5841a8c339/A in TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. 2024-11-10T22:37:54,176 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/3fc52931291045539a3d338411c181bf, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/9c611724e8ce43109bce3190aba006b3, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/24be0ad3d9c64efb9e4053b80350110c] into tmpdir=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp, totalSize=37.1 K 2024-11-10T22:37:54,177 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting db6eccb0b2e54b92b191435288b0b2bb, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=446, earliestPutTs=1731278272127 2024-11-10T22:37:54,178 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3fc52931291045539a3d338411c181bf, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=446, earliestPutTs=1731278272127 2024-11-10T22:37:54,178 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting d583253c5f7245fb88c05fadb1974716, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=474, earliestPutTs=1731278272259 2024-11-10T22:37:54,178 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9c611724e8ce43109bce3190aba006b3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=474, earliestPutTs=1731278272259 2024-11-10T22:37:54,179 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting ad11d27f247c437f83e6bc6dae42b5fb, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=488, earliestPutTs=1731278272945 2024-11-10T22:37:54,179 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 24be0ad3d9c64efb9e4053b80350110c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=488, earliestPutTs=1731278272938 2024-11-10T22:37:54,190 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): dbbcde27e8d425fcc5932f5841a8c339#B#compaction#105 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T22:37:54,190 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/B/f29bd54cfc9945bcb9f9e6c70e947596 is 50, key is test_row_0/B:col10/1731278272945/Put/seqid=0 2024-11-10T22:37:54,193 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): dbbcde27e8d425fcc5932f5841a8c339#A#compaction#106 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T22:37:54,194 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/A/fe72c933b2fa465f8ec0e7e6039bb4c9 is 50, key is test_row_0/A:col10/1731278272945/Put/seqid=0 2024-11-10T22:37:54,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073741942_1118 (size=13493) 2024-11-10T22:37:54,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073741943_1119 (size=13493) 2024-11-10T22:37:54,210 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/A/fe72c933b2fa465f8ec0e7e6039bb4c9 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/fe72c933b2fa465f8ec0e7e6039bb4c9 2024-11-10T22:37:54,218 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dbbcde27e8d425fcc5932f5841a8c339/A of dbbcde27e8d425fcc5932f5841a8c339 into fe72c933b2fa465f8ec0e7e6039bb4c9(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T22:37:54,218 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dbbcde27e8d425fcc5932f5841a8c339: 2024-11-10T22:37:54,218 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., storeName=dbbcde27e8d425fcc5932f5841a8c339/A, priority=13, startTime=1731278274174; duration=0sec 2024-11-10T22:37:54,218 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T22:37:54,218 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dbbcde27e8d425fcc5932f5841a8c339:A 2024-11-10T22:37:54,218 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T22:37:54,219 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35549 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T22:37:54,219 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HStore(1541): dbbcde27e8d425fcc5932f5841a8c339/C is initiating minor compaction (all files) 2024-11-10T22:37:54,220 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dbbcde27e8d425fcc5932f5841a8c339/C in TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. 2024-11-10T22:37:54,220 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/2c424e9f1614487fbebe8b57481cdb54, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/6c10a4657cc14291b13fe1750b2dab2f, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/06eed5fa99724f70af604cb7a95d7903] into tmpdir=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp, totalSize=34.7 K 2024-11-10T22:37:54,220 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2c424e9f1614487fbebe8b57481cdb54, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=446, earliestPutTs=1731278272127 2024-11-10T22:37:54,221 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6c10a4657cc14291b13fe1750b2dab2f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=474, earliestPutTs=1731278272259 2024-11-10T22:37:54,221 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 06eed5fa99724f70af604cb7a95d7903, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=488, earliestPutTs=1731278272945 2024-11-10T22:37:54,228 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): dbbcde27e8d425fcc5932f5841a8c339#C#compaction#107 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T22:37:54,229 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/C/059ad6464df0438eb0b2cd7d679552c4 is 50, key is test_row_0/C:col10/1731278272945/Put/seqid=0 2024-11-10T22:37:54,235 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073741944_1120 (size=13493) 2024-11-10T22:37:54,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-10T22:37:54,273 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T22:37:54,274 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T22:37:54,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(8855): Flush requested on dbbcde27e8d425fcc5932f5841a8c339 2024-11-10T22:37:54,274 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T22:37:54,274 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing dbbcde27e8d425fcc5932f5841a8c339 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-10T22:37:54,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] procedure2.ProcedureExecutor(1139): Stored pid=29, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=29, table=TestAcidGuarantees 2024-11-10T22:37:54,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-10T22:37:54,277 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=29, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=29, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T22:37:54,278 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=29, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=29, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T22:37:54,278 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T22:37:54,282 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/A/891189518e93489b9f14a79c54608f75 is 50, key is test_row_0/A:col10/1731278274165/Put/seqid=0 2024-11-10T22:37:54,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073741945_1121 (size=17181) 2024-11-10T22:37:54,292 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:54,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 292 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43680 deadline: 1731278334287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:54,292 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:54,293 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:54,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43676 deadline: 1731278334288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:54,293 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:54,293 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:54,293 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:54,293 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:54,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43730 deadline: 1731278334289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:54,293 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:54,293 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:54,293 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:54,293 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:54,293 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:54,304 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:54,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43700 deadline: 1731278334302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:54,305 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:54,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43650 deadline: 1731278334302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:54,305 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 849d0d71310f,43753,1731278250663 for 'test_row_0' in dbbcde27e8d425fcc5932f5841a8c339 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8213 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-10T22:37:54,305 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:54,305 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:54,305 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:54,305 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 849d0d71310f,43753,1731278250663 for 'test_row_0' in dbbcde27e8d425fcc5932f5841a8c339 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8215 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-10T22:37:54,306 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:54,306 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:54,306 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:54,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-10T22:37:54,406 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:54,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 293 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43680 deadline: 1731278334402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:54,407 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:54,407 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:54,407 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:54,407 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:54,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43676 deadline: 1731278334402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:54,408 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:54,407 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:54,408 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:54,408 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:54,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43730 deadline: 1731278334402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:54,408 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:54,408 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:54,408 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:54,433 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=30 2024-11-10T22:37:54,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. 2024-11-10T22:37:54,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. as already flushing 2024-11-10T22:37:54,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. 2024-11-10T22:37:54,434 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] handler.RSProcedureHandler(58): pid=30 java.io.IOException: Unable to complete flush {ENCODED => dbbcde27e8d425fcc5932f5841a8c339, NAME => 'TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:37:54,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=30 java.io.IOException: Unable to complete flush {ENCODED => dbbcde27e8d425fcc5932f5841a8c339, NAME => 'TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:37:54,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=30 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dbbcde27e8d425fcc5932f5841a8c339, NAME => 'TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dbbcde27e8d425fcc5932f5841a8c339, NAME => 'TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:37:54,587 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=30 2024-11-10T22:37:54,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. 2024-11-10T22:37:54,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. as already flushing 2024-11-10T22:37:54,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. 2024-11-10T22:37:54,588 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] handler.RSProcedureHandler(58): pid=30 java.io.IOException: Unable to complete flush {ENCODED => dbbcde27e8d425fcc5932f5841a8c339, NAME => 'TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:37:54,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=30 java.io.IOException: Unable to complete flush {ENCODED => dbbcde27e8d425fcc5932f5841a8c339, NAME => 'TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:37:54,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=30 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dbbcde27e8d425fcc5932f5841a8c339, NAME => 'TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dbbcde27e8d425fcc5932f5841a8c339, NAME => 'TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:37:54,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-10T22:37:54,605 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/B/f29bd54cfc9945bcb9f9e6c70e947596 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/f29bd54cfc9945bcb9f9e6c70e947596 2024-11-10T22:37:54,611 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dbbcde27e8d425fcc5932f5841a8c339/B of dbbcde27e8d425fcc5932f5841a8c339 into f29bd54cfc9945bcb9f9e6c70e947596(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T22:37:54,611 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dbbcde27e8d425fcc5932f5841a8c339: 2024-11-10T22:37:54,611 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., storeName=dbbcde27e8d425fcc5932f5841a8c339/B, priority=13, startTime=1731278274174; duration=0sec 2024-11-10T22:37:54,611 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T22:37:54,611 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dbbcde27e8d425fcc5932f5841a8c339:B 2024-11-10T22:37:54,615 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:54,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43676 deadline: 1731278334612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:54,616 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:54,616 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:54,616 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:54,616 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:54,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 294 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43680 deadline: 1731278334612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:54,617 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:54,617 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:54,617 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:54,617 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:54,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43730 deadline: 1731278334613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:54,617 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:54,617 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:54,617 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:54,641 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/C/059ad6464df0438eb0b2cd7d679552c4 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/059ad6464df0438eb0b2cd7d679552c4 2024-11-10T22:37:54,647 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dbbcde27e8d425fcc5932f5841a8c339/C of dbbcde27e8d425fcc5932f5841a8c339 into 059ad6464df0438eb0b2cd7d679552c4(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T22:37:54,647 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dbbcde27e8d425fcc5932f5841a8c339: 2024-11-10T22:37:54,647 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., storeName=dbbcde27e8d425fcc5932f5841a8c339/C, priority=13, startTime=1731278274174; duration=0sec 2024-11-10T22:37:54,648 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T22:37:54,648 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dbbcde27e8d425fcc5932f5841a8c339:C 2024-11-10T22:37:54,690 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=515 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/A/891189518e93489b9f14a79c54608f75 2024-11-10T22:37:54,700 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/B/e129ae36994a4be9880831810b4ed9da is 50, key is test_row_0/B:col10/1731278274165/Put/seqid=0 2024-11-10T22:37:54,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073741946_1122 (size=12301) 2024-11-10T22:37:54,741 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=30 2024-11-10T22:37:54,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. 2024-11-10T22:37:54,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. as already flushing 2024-11-10T22:37:54,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. 2024-11-10T22:37:54,741 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] handler.RSProcedureHandler(58): pid=30 java.io.IOException: Unable to complete flush {ENCODED => dbbcde27e8d425fcc5932f5841a8c339, NAME => 'TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:37:54,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=30 java.io.IOException: Unable to complete flush {ENCODED => dbbcde27e8d425fcc5932f5841a8c339, NAME => 'TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:37:54,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=30 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dbbcde27e8d425fcc5932f5841a8c339, NAME => 'TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dbbcde27e8d425fcc5932f5841a8c339, NAME => 'TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:37:54,894 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=30 2024-11-10T22:37:54,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. 2024-11-10T22:37:54,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. as already flushing 2024-11-10T22:37:54,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. 2024-11-10T22:37:54,895 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] handler.RSProcedureHandler(58): pid=30 java.io.IOException: Unable to complete flush {ENCODED => dbbcde27e8d425fcc5932f5841a8c339, NAME => 'TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:37:54,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=30 java.io.IOException: Unable to complete flush {ENCODED => dbbcde27e8d425fcc5932f5841a8c339, NAME => 'TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:37:54,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=30 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dbbcde27e8d425fcc5932f5841a8c339, NAME => 'TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dbbcde27e8d425fcc5932f5841a8c339, NAME => 'TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:37:54,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-10T22:37:54,924 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:54,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43676 deadline: 1731278334922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:54,925 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:54,925 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:54,925 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:54,925 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:54,925 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:54,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 295 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43680 deadline: 1731278334922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:54,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43730 deadline: 1731278334922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:54,925 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:54,925 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:54,925 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:54,925 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:54,925 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:54,925 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:55,048 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=30 2024-11-10T22:37:55,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. 2024-11-10T22:37:55,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. as already flushing 2024-11-10T22:37:55,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. 2024-11-10T22:37:55,048 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] handler.RSProcedureHandler(58): pid=30 java.io.IOException: Unable to complete flush {ENCODED => dbbcde27e8d425fcc5932f5841a8c339, NAME => 'TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:37:55,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=30 java.io.IOException: Unable to complete flush {ENCODED => dbbcde27e8d425fcc5932f5841a8c339, NAME => 'TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:37:55,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=30 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dbbcde27e8d425fcc5932f5841a8c339, NAME => 'TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dbbcde27e8d425fcc5932f5841a8c339, NAME => 'TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:37:55,111 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=515 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/B/e129ae36994a4be9880831810b4ed9da 2024-11-10T22:37:55,121 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/C/d26787b8090244f0a08df9f661a0ec44 is 50, key is test_row_0/C:col10/1731278274165/Put/seqid=0 2024-11-10T22:37:55,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073741947_1123 (size=12301) 2024-11-10T22:37:55,201 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=30 2024-11-10T22:37:55,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. 2024-11-10T22:37:55,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. as already flushing 2024-11-10T22:37:55,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. 2024-11-10T22:37:55,202 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] handler.RSProcedureHandler(58): pid=30 java.io.IOException: Unable to complete flush {ENCODED => dbbcde27e8d425fcc5932f5841a8c339, NAME => 'TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:37:55,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=30 java.io.IOException: Unable to complete flush {ENCODED => dbbcde27e8d425fcc5932f5841a8c339, NAME => 'TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:37:55,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=30 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dbbcde27e8d425fcc5932f5841a8c339, NAME => 'TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dbbcde27e8d425fcc5932f5841a8c339, NAME => 'TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:37:55,355 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=30 2024-11-10T22:37:55,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. 2024-11-10T22:37:55,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. as already flushing 2024-11-10T22:37:55,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. 2024-11-10T22:37:55,356 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] handler.RSProcedureHandler(58): pid=30 java.io.IOException: Unable to complete flush {ENCODED => dbbcde27e8d425fcc5932f5841a8c339, NAME => 'TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:37:55,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=30 java.io.IOException: Unable to complete flush {ENCODED => dbbcde27e8d425fcc5932f5841a8c339, NAME => 'TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:37:55,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=30 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dbbcde27e8d425fcc5932f5841a8c339, NAME => 'TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dbbcde27e8d425fcc5932f5841a8c339, NAME => 'TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:37:55,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-10T22:37:55,433 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:55,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43676 deadline: 1731278335432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:55,434 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:55,434 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:55,434 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:55,434 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:55,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 296 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43680 deadline: 1731278335432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:55,435 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:55,435 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:55,435 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:55,435 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:55,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43730 deadline: 1731278335432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:55,435 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:55,435 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:55,435 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:55,509 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=30 2024-11-10T22:37:55,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. 2024-11-10T22:37:55,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. as already flushing 2024-11-10T22:37:55,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. 2024-11-10T22:37:55,509 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] handler.RSProcedureHandler(58): pid=30 java.io.IOException: Unable to complete flush {ENCODED => dbbcde27e8d425fcc5932f5841a8c339, NAME => 'TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:37:55,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=30 java.io.IOException: Unable to complete flush {ENCODED => dbbcde27e8d425fcc5932f5841a8c339, NAME => 'TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:37:55,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=30 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dbbcde27e8d425fcc5932f5841a8c339, NAME => 'TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dbbcde27e8d425fcc5932f5841a8c339, NAME => 'TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:37:55,541 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=515 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/C/d26787b8090244f0a08df9f661a0ec44 2024-11-10T22:37:55,547 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/A/891189518e93489b9f14a79c54608f75 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/891189518e93489b9f14a79c54608f75 2024-11-10T22:37:55,552 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/891189518e93489b9f14a79c54608f75, entries=250, sequenceid=515, filesize=16.8 K 2024-11-10T22:37:55,554 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/B/e129ae36994a4be9880831810b4ed9da as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/e129ae36994a4be9880831810b4ed9da 2024-11-10T22:37:55,558 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/e129ae36994a4be9880831810b4ed9da, entries=150, sequenceid=515, filesize=12.0 K 2024-11-10T22:37:55,560 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/C/d26787b8090244f0a08df9f661a0ec44 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/d26787b8090244f0a08df9f661a0ec44 2024-11-10T22:37:55,565 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/d26787b8090244f0a08df9f661a0ec44, entries=150, sequenceid=515, filesize=12.0 K 2024-11-10T22:37:55,566 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for dbbcde27e8d425fcc5932f5841a8c339 in 1292ms, sequenceid=515, compaction requested=false 2024-11-10T22:37:55,566 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for dbbcde27e8d425fcc5932f5841a8c339: 2024-11-10T22:37:55,663 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=30 2024-11-10T22:37:55,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. 2024-11-10T22:37:55,663 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegion(2902): Flushing dbbcde27e8d425fcc5932f5841a8c339 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-10T22:37:55,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/A/83e1c0dae4f54125aa06e31a371e7b8b is 50, key is test_row_0/A:col10/1731278274286/Put/seqid=0 2024-11-10T22:37:55,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073741948_1124 (size=12301) 2024-11-10T22:37:56,074 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=527 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/A/83e1c0dae4f54125aa06e31a371e7b8b 2024-11-10T22:37:56,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/B/e8b00a3a99734ae0b74ba3be1750c5f6 is 50, key is test_row_0/B:col10/1731278274286/Put/seqid=0 2024-11-10T22:37:56,091 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073741949_1125 (size=12301) 2024-11-10T22:37:56,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-10T22:37:56,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(8855): Flush requested on dbbcde27e8d425fcc5932f5841a8c339 2024-11-10T22:37:56,444 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. as already flushing 2024-11-10T22:37:56,485 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:56,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 305 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43680 deadline: 1731278336481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:56,486 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:56,486 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:56,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43730 deadline: 1731278336482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:56,486 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:56,486 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:56,486 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:56,486 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:56,487 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:56,490 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:56,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43676 deadline: 1731278336485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:56,490 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:56,490 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:56,490 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:56,491 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-11-10T22:37:56,491 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T22:37:56,491 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=527 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/B/e8b00a3a99734ae0b74ba3be1750c5f6 2024-11-10T22:37:56,492 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-11-10T22:37:56,492 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T22:37:56,495 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-11-10T22:37:56,495 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T22:37:56,497 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-11-10T22:37:56,497 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T22:37:56,499 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:37:56,499 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:37:56,499 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:37:56,499 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:37:56,499 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:37:56,499 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:37:56,499 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T22:37:56,499 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T22:37:56,500 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T22:37:56,500 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:37:56,500 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:37:56,500 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T22:37:56,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/C/c2e31da1398a4659a956fd2ae75b377a is 50, key is test_row_0/C:col10/1731278274286/Put/seqid=0 2024-11-10T22:37:56,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073741950_1126 (size=12301) 2024-11-10T22:37:56,592 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:56,592 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:56,592 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:56,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43676 deadline: 1731278336592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:56,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 306 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43680 deadline: 1731278336592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:56,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43730 deadline: 1731278336592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:56,593 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:56,593 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:56,593 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:56,593 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:56,593 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:56,593 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:56,593 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:56,593 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:56,593 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:56,803 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:56,803 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:56,803 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:37:56,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43730 deadline: 1731278336803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:56,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43676 deadline: 1731278336803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:56,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 307 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43680 deadline: 1731278336803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 2024-11-10T22:37:56,804 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:56,804 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:56,804 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:56,804 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:56,804 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:56,804 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:37:56,804 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:56,804 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:56,804 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:37:56,906 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=527 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/C/c2e31da1398a4659a956fd2ae75b377a 2024-11-10T22:37:56,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/A/83e1c0dae4f54125aa06e31a371e7b8b as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/83e1c0dae4f54125aa06e31a371e7b8b 2024-11-10T22:37:56,916 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/83e1c0dae4f54125aa06e31a371e7b8b, entries=150, sequenceid=527, filesize=12.0 K 2024-11-10T22:37:56,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/B/e8b00a3a99734ae0b74ba3be1750c5f6 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/e8b00a3a99734ae0b74ba3be1750c5f6 2024-11-10T22:37:56,921 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/e8b00a3a99734ae0b74ba3be1750c5f6, entries=150, sequenceid=527, filesize=12.0 K 2024-11-10T22:37:56,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/C/c2e31da1398a4659a956fd2ae75b377a as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/c2e31da1398a4659a956fd2ae75b377a 2024-11-10T22:37:56,925 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/c2e31da1398a4659a956fd2ae75b377a, entries=150, sequenceid=527, filesize=12.0 K 2024-11-10T22:37:56,926 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=154.31 KB/158010 for dbbcde27e8d425fcc5932f5841a8c339 in 1263ms, sequenceid=527, compaction requested=true 2024-11-10T22:37:56,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegion(2603): Flush status journal for dbbcde27e8d425fcc5932f5841a8c339: 2024-11-10T22:37:56,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. 2024-11-10T22:37:56,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=30 2024-11-10T22:37:56,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.HMaster(4169): Remote procedure done, pid=30 2024-11-10T22:37:56,929 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-11-10T22:37:56,929 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6500 sec 2024-11-10T22:37:56,931 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=29, table=TestAcidGuarantees in 2.6560 sec 2024-11-10T22:37:57,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(8855): Flush requested on dbbcde27e8d425fcc5932f5841a8c339 2024-11-10T22:37:57,114 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T22:37:57,114 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing dbbcde27e8d425fcc5932f5841a8c339 3/3 column families, dataSize=174.43 KB heapSize=457.78 KB 2024-11-10T22:37:57,114 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-11-10T22:37:57,114 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-11-10T22:37:57,114 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-11-10T22:37:57,114 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T22:37:57,114 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T22:37:57,114 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:37:57,114 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T22:37:57,114 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:37:57,114 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:37:57,115 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:37:57,115 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:37:57,115 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:37:57,115 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T22:37:57,115 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T22:37:57,115 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T22:37:57,119 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/A/4b15be1599a24fb29799af074877bbfe is 50, key is test_row_0/A:col10/1731278277112/Put/seqid=0 2024-11-10T22:37:57,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073741951_1127 (size=12301) 2024-11-10T22:37:57,524 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=556 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/A/4b15be1599a24fb29799af074877bbfe 2024-11-10T22:37:57,531 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/B/7717f8fb6403459e9dae4efab1ee93e1 is 50, key is test_row_0/B:col10/1731278277112/Put/seqid=0 2024-11-10T22:37:57,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073741952_1128 (size=12301) 2024-11-10T22:37:57,936 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=556 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/B/7717f8fb6403459e9dae4efab1ee93e1 2024-11-10T22:37:57,944 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/C/46b1c6d9a43b48c39f86135575a61a3e is 50, key is test_row_0/C:col10/1731278277112/Put/seqid=0 2024-11-10T22:37:57,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073741953_1129 (size=12301) 2024-11-10T22:37:58,349 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=556 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/C/46b1c6d9a43b48c39f86135575a61a3e 2024-11-10T22:37:58,354 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/A/4b15be1599a24fb29799af074877bbfe as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/4b15be1599a24fb29799af074877bbfe 2024-11-10T22:37:58,358 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/4b15be1599a24fb29799af074877bbfe, entries=150, sequenceid=556, filesize=12.0 K 2024-11-10T22:37:58,358 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/B/7717f8fb6403459e9dae4efab1ee93e1 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/7717f8fb6403459e9dae4efab1ee93e1 2024-11-10T22:37:58,363 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/7717f8fb6403459e9dae4efab1ee93e1, entries=150, sequenceid=556, filesize=12.0 K 2024-11-10T22:37:58,364 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/C/46b1c6d9a43b48c39f86135575a61a3e as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/46b1c6d9a43b48c39f86135575a61a3e 2024-11-10T22:37:58,368 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/46b1c6d9a43b48c39f86135575a61a3e, entries=150, sequenceid=556, filesize=12.0 K 2024-11-10T22:37:58,369 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~174.43 KB/178620, heapSize ~457.73 KB/468720, currentSize=0 B/0 for dbbcde27e8d425fcc5932f5841a8c339 in 1254ms, sequenceid=556, compaction requested=true 2024-11-10T22:37:58,369 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for dbbcde27e8d425fcc5932f5841a8c339: 2024-11-10T22:37:58,369 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dbbcde27e8d425fcc5932f5841a8c339:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T22:37:58,369 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T22:37:58,369 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dbbcde27e8d425fcc5932f5841a8c339:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T22:37:58,369 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T22:37:58,369 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T22:37:58,369 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T22:37:58,369 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dbbcde27e8d425fcc5932f5841a8c339:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T22:37:58,369 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T22:37:58,370 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50396 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T22:37:58,370 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 55276 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T22:37:58,370 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1541): dbbcde27e8d425fcc5932f5841a8c339/B is initiating minor compaction (all files) 2024-11-10T22:37:58,370 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HStore(1541): dbbcde27e8d425fcc5932f5841a8c339/A is initiating minor compaction (all files) 2024-11-10T22:37:58,370 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dbbcde27e8d425fcc5932f5841a8c339/B in TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. 2024-11-10T22:37:58,370 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dbbcde27e8d425fcc5932f5841a8c339/A in TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. 2024-11-10T22:37:58,370 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/f29bd54cfc9945bcb9f9e6c70e947596, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/e129ae36994a4be9880831810b4ed9da, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/e8b00a3a99734ae0b74ba3be1750c5f6, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/7717f8fb6403459e9dae4efab1ee93e1] into tmpdir=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp, totalSize=49.2 K 2024-11-10T22:37:58,370 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/fe72c933b2fa465f8ec0e7e6039bb4c9, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/891189518e93489b9f14a79c54608f75, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/83e1c0dae4f54125aa06e31a371e7b8b, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/4b15be1599a24fb29799af074877bbfe] into tmpdir=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp, totalSize=54.0 K 2024-11-10T22:37:58,371 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting f29bd54cfc9945bcb9f9e6c70e947596, keycount=150, bloomtype=ROW, size=13.2 K, encoding=NONE, compression=NONE, seqNum=488, earliestPutTs=1731278272275 2024-11-10T22:37:58,371 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting fe72c933b2fa465f8ec0e7e6039bb4c9, keycount=150, bloomtype=ROW, size=13.2 K, encoding=NONE, compression=NONE, seqNum=488, earliestPutTs=1731278272275 2024-11-10T22:37:58,371 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 891189518e93489b9f14a79c54608f75, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=515, earliestPutTs=1731278274155 2024-11-10T22:37:58,371 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting e129ae36994a4be9880831810b4ed9da, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=515, earliestPutTs=1731278274165 2024-11-10T22:37:58,372 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting e8b00a3a99734ae0b74ba3be1750c5f6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=527, earliestPutTs=1731278274285 2024-11-10T22:37:58,372 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 83e1c0dae4f54125aa06e31a371e7b8b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=527, earliestPutTs=1731278274285 2024-11-10T22:37:58,372 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting 7717f8fb6403459e9dae4efab1ee93e1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=556, earliestPutTs=1731278276479 2024-11-10T22:37:58,372 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4b15be1599a24fb29799af074877bbfe, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=556, earliestPutTs=1731278276479 2024-11-10T22:37:58,382 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): dbbcde27e8d425fcc5932f5841a8c339#A#compaction#117 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T22:37:58,383 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): dbbcde27e8d425fcc5932f5841a8c339#B#compaction#118 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T22:37:58,383 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/A/1f88a6717d8246cea5dc89b7ece4eb03 is 50, key is test_row_0/A:col10/1731278277112/Put/seqid=0 2024-11-10T22:37:58,384 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/B/1aec3888fbf4467883d12854e34a360c is 50, key is test_row_0/B:col10/1731278277112/Put/seqid=0 2024-11-10T22:37:58,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073741954_1130 (size=13629) 2024-11-10T22:37:58,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073741955_1131 (size=13629) 2024-11-10T22:37:58,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-10T22:37:58,442 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T22:37:58,794 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/B/1aec3888fbf4467883d12854e34a360c as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/1aec3888fbf4467883d12854e34a360c 2024-11-10T22:37:58,794 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/A/1f88a6717d8246cea5dc89b7ece4eb03 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/1f88a6717d8246cea5dc89b7ece4eb03 2024-11-10T22:37:58,799 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in dbbcde27e8d425fcc5932f5841a8c339/A of dbbcde27e8d425fcc5932f5841a8c339 into 1f88a6717d8246cea5dc89b7ece4eb03(size=13.3 K), total size for store is 13.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T22:37:58,799 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in dbbcde27e8d425fcc5932f5841a8c339/B of dbbcde27e8d425fcc5932f5841a8c339 into 1aec3888fbf4467883d12854e34a360c(size=13.3 K), total size for store is 13.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T22:37:58,799 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dbbcde27e8d425fcc5932f5841a8c339: 2024-11-10T22:37:58,799 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dbbcde27e8d425fcc5932f5841a8c339: 2024-11-10T22:37:58,799 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., storeName=dbbcde27e8d425fcc5932f5841a8c339/A, priority=12, startTime=1731278278369; duration=0sec 2024-11-10T22:37:58,799 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., storeName=dbbcde27e8d425fcc5932f5841a8c339/B, priority=12, startTime=1731278278369; duration=0sec 2024-11-10T22:37:58,799 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T22:37:58,799 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dbbcde27e8d425fcc5932f5841a8c339:A 2024-11-10T22:37:58,799 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T22:37:58,799 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dbbcde27e8d425fcc5932f5841a8c339:B 2024-11-10T22:37:58,799 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T22:37:58,800 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50396 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T22:37:58,801 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HStore(1541): dbbcde27e8d425fcc5932f5841a8c339/C is initiating minor compaction (all files) 2024-11-10T22:37:58,801 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dbbcde27e8d425fcc5932f5841a8c339/C in TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. 2024-11-10T22:37:58,801 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/059ad6464df0438eb0b2cd7d679552c4, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/d26787b8090244f0a08df9f661a0ec44, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/c2e31da1398a4659a956fd2ae75b377a, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/46b1c6d9a43b48c39f86135575a61a3e] into tmpdir=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp, totalSize=49.2 K 2024-11-10T22:37:58,801 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 059ad6464df0438eb0b2cd7d679552c4, keycount=150, bloomtype=ROW, size=13.2 K, encoding=NONE, compression=NONE, seqNum=488, earliestPutTs=1731278272275 2024-11-10T22:37:58,801 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting d26787b8090244f0a08df9f661a0ec44, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=515, earliestPutTs=1731278274165 2024-11-10T22:37:58,802 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting c2e31da1398a4659a956fd2ae75b377a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=527, earliestPutTs=1731278274285 2024-11-10T22:37:58,802 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 46b1c6d9a43b48c39f86135575a61a3e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=556, earliestPutTs=1731278276479 2024-11-10T22:37:58,810 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): dbbcde27e8d425fcc5932f5841a8c339#C#compaction#119 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T22:37:58,811 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/C/eaeaf8fcffe04830888ef5b4f46b13cf is 50, key is test_row_0/C:col10/1731278277112/Put/seqid=0 2024-11-10T22:37:58,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073741956_1132 (size=13629) 2024-11-10T22:37:59,063 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-10T22:37:59,220 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/C/eaeaf8fcffe04830888ef5b4f46b13cf as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/eaeaf8fcffe04830888ef5b4f46b13cf 2024-11-10T22:37:59,225 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in dbbcde27e8d425fcc5932f5841a8c339/C of dbbcde27e8d425fcc5932f5841a8c339 into eaeaf8fcffe04830888ef5b4f46b13cf(size=13.3 K), total size for store is 13.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T22:37:59,225 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dbbcde27e8d425fcc5932f5841a8c339: 2024-11-10T22:37:59,225 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., storeName=dbbcde27e8d425fcc5932f5841a8c339/C, priority=12, startTime=1731278278369; duration=0sec 2024-11-10T22:37:59,225 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T22:37:59,225 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dbbcde27e8d425fcc5932f5841a8c339:C 2024-11-10T22:38:04,354 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-11-10T22:38:04,354 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T22:38:04,354 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:38:04,354 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:38:04,354 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T22:38:04,374 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-11-10T22:38:04,374 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T22:38:04,374 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:38:04,374 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:38:04,374 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T22:38:04,374 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-10T22:38:04,374 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 56 2024-11-10T22:38:04,374 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 235 2024-11-10T22:38:04,374 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 24 2024-11-10T22:38:04,374 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 62 2024-11-10T22:38:04,374 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 62 2024-11-10T22:38:04,374 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-10T22:38:04,374 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3528 2024-11-10T22:38:04,374 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3651 2024-11-10T22:38:04,374 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-10T22:38:04,375 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1572 2024-11-10T22:38:04,375 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4711 rows 2024-11-10T22:38:04,375 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1575 2024-11-10T22:38:04,375 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4721 rows 2024-11-10T22:38:04,375 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-10T22:38:04,375 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T22:38:04,375 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:38:04,375 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:38:04,375 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T22:38:04,380 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-10T22:38:04,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] procedure2.ProcedureExecutor(1139): Stored pid=31, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-10T22:38:04,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-10T22:38:04,389 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731278284388"}]},"ts":"1731278284388"} 2024-11-10T22:38:04,391 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-10T22:38:04,391 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-10T22:38:04,392 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-10T22:38:04,396 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=dbbcde27e8d425fcc5932f5841a8c339, UNASSIGN}] 2024-11-10T22:38:04,397 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=dbbcde27e8d425fcc5932f5841a8c339, UNASSIGN 2024-11-10T22:38:04,398 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=dbbcde27e8d425fcc5932f5841a8c339, regionState=CLOSING, regionLocation=849d0d71310f,43753,1731278250663 2024-11-10T22:38:04,400 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=dbbcde27e8d425fcc5932f5841a8c339, UNASSIGN because future has completed 2024-11-10T22:38:04,401 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-10T22:38:04,401 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=33, state=RUNNABLE, hasLock=false; CloseRegionProcedure dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663}] 2024-11-10T22:38:04,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-10T22:38:04,557 INFO [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] handler.UnassignRegionHandler(122): Close dbbcde27e8d425fcc5932f5841a8c339 2024-11-10T22:38:04,558 DEBUG [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-10T22:38:04,558 DEBUG [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1722): Closing dbbcde27e8d425fcc5932f5841a8c339, disabling compactions & flushes 2024-11-10T22:38:04,558 INFO [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. 2024-11-10T22:38:04,558 DEBUG [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. 2024-11-10T22:38:04,558 DEBUG [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. after waiting 0 ms 2024-11-10T22:38:04,558 DEBUG [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. 2024-11-10T22:38:04,559 INFO [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(2902): Flushing dbbcde27e8d425fcc5932f5841a8c339 3/3 column families, dataSize=13.42 KB heapSize=35.91 KB 2024-11-10T22:38:04,563 DEBUG [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/A/7594984ef5974a20a2a5f771902ac65c is 50, key is test_row_0/A:col10/1731278284372/Put/seqid=0 2024-11-10T22:38:04,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073741957_1133 (size=7415) 2024-11-10T22:38:04,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-10T22:38:04,968 INFO [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=564 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/A/7594984ef5974a20a2a5f771902ac65c 2024-11-10T22:38:04,976 DEBUG [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/B/c15eb60719d448078476c2b8bb0b5c73 is 50, key is test_row_0/B:col10/1731278284372/Put/seqid=0 2024-11-10T22:38:04,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073741958_1134 (size=7415) 2024-11-10T22:38:05,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-10T22:38:05,380 INFO [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=564 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/B/c15eb60719d448078476c2b8bb0b5c73 2024-11-10T22:38:05,387 DEBUG [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/C/b85a88b1ae96468fab19983605271202 is 50, key is test_row_0/C:col10/1731278284372/Put/seqid=0 2024-11-10T22:38:05,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073741959_1135 (size=7415) 2024-11-10T22:38:05,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-10T22:38:05,791 INFO [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=564 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/C/b85a88b1ae96468fab19983605271202 2024-11-10T22:38:05,796 DEBUG [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/A/7594984ef5974a20a2a5f771902ac65c as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/7594984ef5974a20a2a5f771902ac65c 2024-11-10T22:38:05,801 INFO [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/7594984ef5974a20a2a5f771902ac65c, entries=50, sequenceid=564, filesize=7.2 K 2024-11-10T22:38:05,801 DEBUG [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/B/c15eb60719d448078476c2b8bb0b5c73 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/c15eb60719d448078476c2b8bb0b5c73 2024-11-10T22:38:05,805 INFO [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/c15eb60719d448078476c2b8bb0b5c73, entries=50, sequenceid=564, filesize=7.2 K 2024-11-10T22:38:05,806 DEBUG [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/.tmp/C/b85a88b1ae96468fab19983605271202 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/b85a88b1ae96468fab19983605271202 2024-11-10T22:38:05,810 INFO [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/b85a88b1ae96468fab19983605271202, entries=50, sequenceid=564, filesize=7.2 K 2024-11-10T22:38:05,810 INFO [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(3140): Finished flush of dataSize ~13.42 KB/13740, heapSize ~35.86 KB/36720, currentSize=0 B/0 for dbbcde27e8d425fcc5932f5841a8c339 in 1252ms, sequenceid=564, compaction requested=false 2024-11-10T22:38:05,811 DEBUG [StoreCloser-TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/1189fa75ce874ac6bb646298bd75533d, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/0c429c842d264d72ac93837d08e0e7fa, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/30db2d12e41c44208f7fe2a15c2349d8, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/404499702fea43888d35b759c7a99f88, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/771a9c58cabc4d4fbd00595c0c7c0174, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/1b77b45ec7a842b3b3e8dc9f88ee636c, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/6774ce93b7c34fb5a95373a28079588d, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/b1e8feacc0fc4dec8f5885dfaf3ec41f, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/efb438a3fe764ceb913da2475ba4cc2d, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/7773dcf022a942cb998aeaf04c624f2e, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/3a4ed58ed2ad4752921be7d8438bc1df, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/ef2cc67048ba48e38466d0cff4ed3acb, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/1289798ebb01461089cd1f30d05781fb, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/f22dacb7a9344f8cb06470098dd57b3e, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/8a3d6180dc9947098b88da50e9ed8ebf, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/8a3546d820424d87a75ed1e02e82c478, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/7ed846158b1e43c3880b1358d6ba8941, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/32d409df8d524c90821cf8dab8e5f762, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/26047603692f40d3bd5b6c269bfc0dbc, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/45c145cc147c4c8e8d16c288e34e57f4, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/de333a7a42014b6c906b6ef354f9f88f, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/8c7765598d6c4c6e82a8612db5b51cfd, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/f8b9838d86664305a0c7ba6d38c86560, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/2b6c1d275ca44d89ac048b8ec3e154ad, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/76c2ae53e62a467e9b43a29de62f5918, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/9c21f39a2b9b4aa8bd9dbd725423bf4d, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/d62a206039c5462190da54367ea25796, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/996ca5c651224ad7af261f58bd38f182, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/4612de31b24f4f21be666c4370008fb4, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/ce40513cc5d04cf68bb715a035dcf91a, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/d9e1225883b54667b5752aeb9c01cd68, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/e20c26f86cd14d2e88445aad7199e94b, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/3fc52931291045539a3d338411c181bf, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/9c611724e8ce43109bce3190aba006b3, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/fe72c933b2fa465f8ec0e7e6039bb4c9, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/24be0ad3d9c64efb9e4053b80350110c, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/891189518e93489b9f14a79c54608f75, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/83e1c0dae4f54125aa06e31a371e7b8b, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/4b15be1599a24fb29799af074877bbfe] to archive 2024-11-10T22:38:05,814 DEBUG [StoreCloser-TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T22:38:05,819 DEBUG [StoreCloser-TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/1189fa75ce874ac6bb646298bd75533d to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/1189fa75ce874ac6bb646298bd75533d 2024-11-10T22:38:05,821 DEBUG [StoreCloser-TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/0c429c842d264d72ac93837d08e0e7fa to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/0c429c842d264d72ac93837d08e0e7fa 2024-11-10T22:38:05,822 DEBUG [StoreCloser-TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/30db2d12e41c44208f7fe2a15c2349d8 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/30db2d12e41c44208f7fe2a15c2349d8 2024-11-10T22:38:05,823 DEBUG [StoreCloser-TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/404499702fea43888d35b759c7a99f88 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/404499702fea43888d35b759c7a99f88 2024-11-10T22:38:05,825 DEBUG [StoreCloser-TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/771a9c58cabc4d4fbd00595c0c7c0174 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/771a9c58cabc4d4fbd00595c0c7c0174 2024-11-10T22:38:05,826 DEBUG [StoreCloser-TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/1b77b45ec7a842b3b3e8dc9f88ee636c to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/1b77b45ec7a842b3b3e8dc9f88ee636c 2024-11-10T22:38:05,827 DEBUG [StoreCloser-TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/6774ce93b7c34fb5a95373a28079588d to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/6774ce93b7c34fb5a95373a28079588d 2024-11-10T22:38:05,829 DEBUG [StoreCloser-TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/b1e8feacc0fc4dec8f5885dfaf3ec41f to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/b1e8feacc0fc4dec8f5885dfaf3ec41f 2024-11-10T22:38:05,830 DEBUG [StoreCloser-TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/efb438a3fe764ceb913da2475ba4cc2d to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/efb438a3fe764ceb913da2475ba4cc2d 2024-11-10T22:38:05,831 DEBUG [StoreCloser-TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/7773dcf022a942cb998aeaf04c624f2e to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/7773dcf022a942cb998aeaf04c624f2e 2024-11-10T22:38:05,832 DEBUG [StoreCloser-TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/3a4ed58ed2ad4752921be7d8438bc1df to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/3a4ed58ed2ad4752921be7d8438bc1df 2024-11-10T22:38:05,833 DEBUG [StoreCloser-TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/ef2cc67048ba48e38466d0cff4ed3acb to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/ef2cc67048ba48e38466d0cff4ed3acb 2024-11-10T22:38:05,834 DEBUG [StoreCloser-TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/1289798ebb01461089cd1f30d05781fb to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/1289798ebb01461089cd1f30d05781fb 2024-11-10T22:38:05,835 DEBUG [StoreCloser-TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/f22dacb7a9344f8cb06470098dd57b3e to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/f22dacb7a9344f8cb06470098dd57b3e 2024-11-10T22:38:05,836 DEBUG [StoreCloser-TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/8a3d6180dc9947098b88da50e9ed8ebf to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/8a3d6180dc9947098b88da50e9ed8ebf 2024-11-10T22:38:05,837 DEBUG [StoreCloser-TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/8a3546d820424d87a75ed1e02e82c478 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/8a3546d820424d87a75ed1e02e82c478 2024-11-10T22:38:05,838 DEBUG [StoreCloser-TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/7ed846158b1e43c3880b1358d6ba8941 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/7ed846158b1e43c3880b1358d6ba8941 2024-11-10T22:38:05,839 DEBUG [StoreCloser-TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/32d409df8d524c90821cf8dab8e5f762 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/32d409df8d524c90821cf8dab8e5f762 2024-11-10T22:38:05,840 DEBUG [StoreCloser-TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/26047603692f40d3bd5b6c269bfc0dbc to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/26047603692f40d3bd5b6c269bfc0dbc 2024-11-10T22:38:05,841 DEBUG [StoreCloser-TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/45c145cc147c4c8e8d16c288e34e57f4 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/45c145cc147c4c8e8d16c288e34e57f4 2024-11-10T22:38:05,843 DEBUG [StoreCloser-TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/de333a7a42014b6c906b6ef354f9f88f to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/de333a7a42014b6c906b6ef354f9f88f 2024-11-10T22:38:05,844 DEBUG [StoreCloser-TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/8c7765598d6c4c6e82a8612db5b51cfd to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/8c7765598d6c4c6e82a8612db5b51cfd 2024-11-10T22:38:05,845 DEBUG [StoreCloser-TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/f8b9838d86664305a0c7ba6d38c86560 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/f8b9838d86664305a0c7ba6d38c86560 2024-11-10T22:38:05,846 DEBUG [StoreCloser-TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/2b6c1d275ca44d89ac048b8ec3e154ad to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/2b6c1d275ca44d89ac048b8ec3e154ad 2024-11-10T22:38:05,847 DEBUG [StoreCloser-TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/76c2ae53e62a467e9b43a29de62f5918 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/76c2ae53e62a467e9b43a29de62f5918 2024-11-10T22:38:05,848 DEBUG [StoreCloser-TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/9c21f39a2b9b4aa8bd9dbd725423bf4d to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/9c21f39a2b9b4aa8bd9dbd725423bf4d 2024-11-10T22:38:05,849 DEBUG [StoreCloser-TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/d62a206039c5462190da54367ea25796 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/d62a206039c5462190da54367ea25796 2024-11-10T22:38:05,850 DEBUG [StoreCloser-TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/996ca5c651224ad7af261f58bd38f182 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/996ca5c651224ad7af261f58bd38f182 2024-11-10T22:38:05,851 DEBUG [StoreCloser-TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/4612de31b24f4f21be666c4370008fb4 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/4612de31b24f4f21be666c4370008fb4 2024-11-10T22:38:05,852 DEBUG [StoreCloser-TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/ce40513cc5d04cf68bb715a035dcf91a to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/ce40513cc5d04cf68bb715a035dcf91a 2024-11-10T22:38:05,853 DEBUG [StoreCloser-TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/d9e1225883b54667b5752aeb9c01cd68 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/d9e1225883b54667b5752aeb9c01cd68 2024-11-10T22:38:05,854 DEBUG [StoreCloser-TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/e20c26f86cd14d2e88445aad7199e94b to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/e20c26f86cd14d2e88445aad7199e94b 2024-11-10T22:38:05,855 DEBUG [StoreCloser-TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/3fc52931291045539a3d338411c181bf to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/3fc52931291045539a3d338411c181bf 2024-11-10T22:38:05,856 DEBUG [StoreCloser-TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/9c611724e8ce43109bce3190aba006b3 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/9c611724e8ce43109bce3190aba006b3 2024-11-10T22:38:05,857 DEBUG [StoreCloser-TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/fe72c933b2fa465f8ec0e7e6039bb4c9 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/fe72c933b2fa465f8ec0e7e6039bb4c9 2024-11-10T22:38:05,858 DEBUG [StoreCloser-TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/24be0ad3d9c64efb9e4053b80350110c to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/24be0ad3d9c64efb9e4053b80350110c 2024-11-10T22:38:05,859 DEBUG [StoreCloser-TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/891189518e93489b9f14a79c54608f75 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/891189518e93489b9f14a79c54608f75 2024-11-10T22:38:05,860 DEBUG [StoreCloser-TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/83e1c0dae4f54125aa06e31a371e7b8b to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/83e1c0dae4f54125aa06e31a371e7b8b 2024-11-10T22:38:05,861 DEBUG [StoreCloser-TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/4b15be1599a24fb29799af074877bbfe to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/4b15be1599a24fb29799af074877bbfe 2024-11-10T22:38:05,875 DEBUG [StoreCloser-TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/5cd0ad941b914803bea6c3d527bf4359, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/c6535186dc4d483d87772535592b4b05, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/4aa024b221a84c3295da743597124bd0, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/ee95bc86ea854c3dbcd20fca6ad8dcde, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/569791a5aa134c959326822d6dd5c0df, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/dd73abff494245979542c0db14f502b3, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/7db38536c6e9485896cc5532eefca58a, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/ba7b11a739fc43c895618c1cfcdb9f7b, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/abb0891289fd47348f33a00617065baf, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/c4f5736ecbb5464ca12a45bcdff82990, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/18e5803b2cec4f73964080a404e80f71, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/3e61a0a7018b4fe1bbe6a91dcefebe63, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/17a8fce7eec2421e973126a139aa71cb, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/86a82ed3b0f849aabba7fffe2165c87b, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/b37c539d162346e9af554d094255a24c, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/f9694985fbdd4980a05aae5d3b026c60, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/993ae27b597f4858af80e4559f6bca2a, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/a9e5790d43df46ac94689c886f408356, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/f34a46391854427ea79bb8f7a2ffdd6c, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/5a702cd477014e83a52a63f953db760f, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/2d73eff4c95e4fa59b29e395d886f941, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/947a5f8ac53d450992c6ee072d14bf78, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/bc85c7e6cfd943e79b1463e6784f0663, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/1d75f11d9d2d45fd876df00c6139f638, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/618e4af2aa204aebbbddac8ebf481b65, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/022567373b454e43a2dfb3e0adf279f9, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/6ea9d03eccff41fbb6a7c1c46f03a547, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/82cbd36f92134060a4dfe9bd76e502bd, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/fef6efd9f5e044c082ad868c095c4a51, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/e84d8ff86b6840419c0a17a80f799531, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/9739cb02b5094287abaa031d46627bed, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/db6eccb0b2e54b92b191435288b0b2bb, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/77c78595567b4faa8ee7b409917fa3d3, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/d583253c5f7245fb88c05fadb1974716, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/f29bd54cfc9945bcb9f9e6c70e947596, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/ad11d27f247c437f83e6bc6dae42b5fb, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/e129ae36994a4be9880831810b4ed9da, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/e8b00a3a99734ae0b74ba3be1750c5f6, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/7717f8fb6403459e9dae4efab1ee93e1] to archive 2024-11-10T22:38:05,876 DEBUG [StoreCloser-TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T22:38:05,878 DEBUG [StoreCloser-TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/5cd0ad941b914803bea6c3d527bf4359 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/5cd0ad941b914803bea6c3d527bf4359 2024-11-10T22:38:05,879 DEBUG [StoreCloser-TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/c6535186dc4d483d87772535592b4b05 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/c6535186dc4d483d87772535592b4b05 2024-11-10T22:38:05,880 DEBUG [StoreCloser-TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/4aa024b221a84c3295da743597124bd0 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/4aa024b221a84c3295da743597124bd0 2024-11-10T22:38:05,881 DEBUG [StoreCloser-TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/ee95bc86ea854c3dbcd20fca6ad8dcde to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/ee95bc86ea854c3dbcd20fca6ad8dcde 2024-11-10T22:38:05,882 DEBUG [StoreCloser-TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/569791a5aa134c959326822d6dd5c0df to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/569791a5aa134c959326822d6dd5c0df 2024-11-10T22:38:05,883 DEBUG [StoreCloser-TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/dd73abff494245979542c0db14f502b3 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/dd73abff494245979542c0db14f502b3 2024-11-10T22:38:05,884 DEBUG [StoreCloser-TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/7db38536c6e9485896cc5532eefca58a to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/7db38536c6e9485896cc5532eefca58a 2024-11-10T22:38:05,886 DEBUG [StoreCloser-TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/ba7b11a739fc43c895618c1cfcdb9f7b to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/ba7b11a739fc43c895618c1cfcdb9f7b 2024-11-10T22:38:05,887 DEBUG [StoreCloser-TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/abb0891289fd47348f33a00617065baf to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/abb0891289fd47348f33a00617065baf 2024-11-10T22:38:05,888 DEBUG [StoreCloser-TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/c4f5736ecbb5464ca12a45bcdff82990 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/c4f5736ecbb5464ca12a45bcdff82990 2024-11-10T22:38:05,889 DEBUG [StoreCloser-TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/18e5803b2cec4f73964080a404e80f71 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/18e5803b2cec4f73964080a404e80f71 2024-11-10T22:38:05,890 DEBUG [StoreCloser-TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/3e61a0a7018b4fe1bbe6a91dcefebe63 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/3e61a0a7018b4fe1bbe6a91dcefebe63 2024-11-10T22:38:05,891 DEBUG [StoreCloser-TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/17a8fce7eec2421e973126a139aa71cb to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/17a8fce7eec2421e973126a139aa71cb 2024-11-10T22:38:05,893 DEBUG [StoreCloser-TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/86a82ed3b0f849aabba7fffe2165c87b to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/86a82ed3b0f849aabba7fffe2165c87b 2024-11-10T22:38:05,894 DEBUG [StoreCloser-TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/b37c539d162346e9af554d094255a24c to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/b37c539d162346e9af554d094255a24c 2024-11-10T22:38:05,895 DEBUG [StoreCloser-TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/f9694985fbdd4980a05aae5d3b026c60 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/f9694985fbdd4980a05aae5d3b026c60 2024-11-10T22:38:05,896 DEBUG [StoreCloser-TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/993ae27b597f4858af80e4559f6bca2a to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/993ae27b597f4858af80e4559f6bca2a 2024-11-10T22:38:05,897 DEBUG [StoreCloser-TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/a9e5790d43df46ac94689c886f408356 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/a9e5790d43df46ac94689c886f408356 2024-11-10T22:38:05,898 DEBUG [StoreCloser-TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/f34a46391854427ea79bb8f7a2ffdd6c to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/f34a46391854427ea79bb8f7a2ffdd6c 2024-11-10T22:38:05,899 DEBUG [StoreCloser-TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/5a702cd477014e83a52a63f953db760f to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/5a702cd477014e83a52a63f953db760f 2024-11-10T22:38:05,901 DEBUG [StoreCloser-TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/2d73eff4c95e4fa59b29e395d886f941 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/2d73eff4c95e4fa59b29e395d886f941 2024-11-10T22:38:05,902 DEBUG [StoreCloser-TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/947a5f8ac53d450992c6ee072d14bf78 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/947a5f8ac53d450992c6ee072d14bf78 2024-11-10T22:38:05,903 DEBUG [StoreCloser-TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/bc85c7e6cfd943e79b1463e6784f0663 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/bc85c7e6cfd943e79b1463e6784f0663 2024-11-10T22:38:05,904 DEBUG [StoreCloser-TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/1d75f11d9d2d45fd876df00c6139f638 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/1d75f11d9d2d45fd876df00c6139f638 2024-11-10T22:38:05,905 DEBUG [StoreCloser-TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/618e4af2aa204aebbbddac8ebf481b65 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/618e4af2aa204aebbbddac8ebf481b65 2024-11-10T22:38:05,907 DEBUG [StoreCloser-TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/022567373b454e43a2dfb3e0adf279f9 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/022567373b454e43a2dfb3e0adf279f9 2024-11-10T22:38:05,908 DEBUG [StoreCloser-TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/6ea9d03eccff41fbb6a7c1c46f03a547 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/6ea9d03eccff41fbb6a7c1c46f03a547 2024-11-10T22:38:05,909 DEBUG [StoreCloser-TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/82cbd36f92134060a4dfe9bd76e502bd to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/82cbd36f92134060a4dfe9bd76e502bd 2024-11-10T22:38:05,911 DEBUG [StoreCloser-TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/fef6efd9f5e044c082ad868c095c4a51 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/fef6efd9f5e044c082ad868c095c4a51 2024-11-10T22:38:05,912 DEBUG [StoreCloser-TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/e84d8ff86b6840419c0a17a80f799531 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/e84d8ff86b6840419c0a17a80f799531 2024-11-10T22:38:05,913 DEBUG [StoreCloser-TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/9739cb02b5094287abaa031d46627bed to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/9739cb02b5094287abaa031d46627bed 2024-11-10T22:38:05,914 DEBUG [StoreCloser-TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/db6eccb0b2e54b92b191435288b0b2bb to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/db6eccb0b2e54b92b191435288b0b2bb 2024-11-10T22:38:05,915 DEBUG [StoreCloser-TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/77c78595567b4faa8ee7b409917fa3d3 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/77c78595567b4faa8ee7b409917fa3d3 2024-11-10T22:38:05,916 DEBUG [StoreCloser-TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/d583253c5f7245fb88c05fadb1974716 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/d583253c5f7245fb88c05fadb1974716 2024-11-10T22:38:05,917 DEBUG [StoreCloser-TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/f29bd54cfc9945bcb9f9e6c70e947596 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/f29bd54cfc9945bcb9f9e6c70e947596 2024-11-10T22:38:05,919 DEBUG [StoreCloser-TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/ad11d27f247c437f83e6bc6dae42b5fb to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/ad11d27f247c437f83e6bc6dae42b5fb 2024-11-10T22:38:05,920 DEBUG [StoreCloser-TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/e129ae36994a4be9880831810b4ed9da to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/e129ae36994a4be9880831810b4ed9da 2024-11-10T22:38:05,921 DEBUG [StoreCloser-TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/e8b00a3a99734ae0b74ba3be1750c5f6 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/e8b00a3a99734ae0b74ba3be1750c5f6 2024-11-10T22:38:05,922 DEBUG [StoreCloser-TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/7717f8fb6403459e9dae4efab1ee93e1 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/7717f8fb6403459e9dae4efab1ee93e1 2024-11-10T22:38:05,924 DEBUG [StoreCloser-TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/b9fe603f49564bc099d73affdf19ee8a, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/255cc363969e44c4b682398a1c2fe1b6, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/56020c56435d4253b31b6f1b7d5ab67f, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/abbc714f137140b18a2e2ba4af9fb5dc, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/7769fbdd99834f5f97f2cccd1f0537f6, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/37d709dc3b50494aacc564500dc132db, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/b90640fb49c34a72adbca93d5a248290, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/fd92350c73894ddb86b09d65c32b8961, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/c46c520081464d26a8004e1bdb0f6a89, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/c864390fb72e478c92394a123356081e, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/5ae989950a464c92862eb2a0763892a3, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/9c080a141597400dbd85f66394618e41, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/59d76567a0484d10bdf7bce2a0afea46, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/2f7f3216e5a74c71bc88d4cafd575de7, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/efbf8d6c171d45b08fa5e22a765a3098, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/c3bc94638ef44755b83a5c137a1b4f2a, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/9d4c5f9d3ec94449b3a78127adcd1c6d, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/fa100c3ac50d4b3babf9c1cb9ecca7aa, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/06ffbce4700542529d0c342773a609f7, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/ba4b6037e90c4de184c0f0be2d1df78d, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/7f6b8c49516c43ed8259876beab13e46, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/7c2bc1b4217140ebabd5eb401f1075ea, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/c150439f212f4834afc76f7dc510ee6b, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/dbadc1b2c799480b911a9d5c3c8c5044, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/02dffc988c654acdb7c5b6636b1f8fb0, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/2d8d0d7d82954738a297b75d445597be, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/41a3911cc8004669b1827020c2992317, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/b895e2363bde4657b03f95790ca56414, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/3ccf43dd083747d5b1a8f0ff6607ff79, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/6ce20a8e7b37485d9f4316f87fe029fb, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/57f2f4f1eba44484a47acaf13112c371, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/2c424e9f1614487fbebe8b57481cdb54, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/f533c10058cf43c1996bdbfba5f17afe, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/6c10a4657cc14291b13fe1750b2dab2f, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/059ad6464df0438eb0b2cd7d679552c4, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/06eed5fa99724f70af604cb7a95d7903, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/d26787b8090244f0a08df9f661a0ec44, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/c2e31da1398a4659a956fd2ae75b377a, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/46b1c6d9a43b48c39f86135575a61a3e] to archive 2024-11-10T22:38:05,925 DEBUG [StoreCloser-TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T22:38:05,926 DEBUG [StoreCloser-TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/b9fe603f49564bc099d73affdf19ee8a to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/b9fe603f49564bc099d73affdf19ee8a 2024-11-10T22:38:05,927 DEBUG [StoreCloser-TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/255cc363969e44c4b682398a1c2fe1b6 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/255cc363969e44c4b682398a1c2fe1b6 2024-11-10T22:38:05,929 DEBUG [StoreCloser-TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/56020c56435d4253b31b6f1b7d5ab67f to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/56020c56435d4253b31b6f1b7d5ab67f 2024-11-10T22:38:05,930 DEBUG [StoreCloser-TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/abbc714f137140b18a2e2ba4af9fb5dc to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/abbc714f137140b18a2e2ba4af9fb5dc 2024-11-10T22:38:05,931 DEBUG [StoreCloser-TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/7769fbdd99834f5f97f2cccd1f0537f6 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/7769fbdd99834f5f97f2cccd1f0537f6 2024-11-10T22:38:05,932 DEBUG [StoreCloser-TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/37d709dc3b50494aacc564500dc132db to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/37d709dc3b50494aacc564500dc132db 2024-11-10T22:38:05,933 DEBUG [StoreCloser-TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/b90640fb49c34a72adbca93d5a248290 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/b90640fb49c34a72adbca93d5a248290 2024-11-10T22:38:05,934 DEBUG [StoreCloser-TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/fd92350c73894ddb86b09d65c32b8961 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/fd92350c73894ddb86b09d65c32b8961 2024-11-10T22:38:05,935 DEBUG [StoreCloser-TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/c46c520081464d26a8004e1bdb0f6a89 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/c46c520081464d26a8004e1bdb0f6a89 2024-11-10T22:38:05,936 DEBUG [StoreCloser-TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/c864390fb72e478c92394a123356081e to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/c864390fb72e478c92394a123356081e 2024-11-10T22:38:05,937 DEBUG [StoreCloser-TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/5ae989950a464c92862eb2a0763892a3 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/5ae989950a464c92862eb2a0763892a3 2024-11-10T22:38:05,939 DEBUG [StoreCloser-TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/9c080a141597400dbd85f66394618e41 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/9c080a141597400dbd85f66394618e41 2024-11-10T22:38:05,940 DEBUG [StoreCloser-TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/59d76567a0484d10bdf7bce2a0afea46 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/59d76567a0484d10bdf7bce2a0afea46 2024-11-10T22:38:05,941 DEBUG [StoreCloser-TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/2f7f3216e5a74c71bc88d4cafd575de7 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/2f7f3216e5a74c71bc88d4cafd575de7 2024-11-10T22:38:05,942 DEBUG [StoreCloser-TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/efbf8d6c171d45b08fa5e22a765a3098 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/efbf8d6c171d45b08fa5e22a765a3098 2024-11-10T22:38:05,943 DEBUG [StoreCloser-TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/c3bc94638ef44755b83a5c137a1b4f2a to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/c3bc94638ef44755b83a5c137a1b4f2a 2024-11-10T22:38:05,944 DEBUG [StoreCloser-TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/9d4c5f9d3ec94449b3a78127adcd1c6d to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/9d4c5f9d3ec94449b3a78127adcd1c6d 2024-11-10T22:38:05,946 DEBUG [StoreCloser-TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/fa100c3ac50d4b3babf9c1cb9ecca7aa to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/fa100c3ac50d4b3babf9c1cb9ecca7aa 2024-11-10T22:38:05,947 DEBUG [StoreCloser-TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/06ffbce4700542529d0c342773a609f7 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/06ffbce4700542529d0c342773a609f7 2024-11-10T22:38:05,948 DEBUG [StoreCloser-TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/ba4b6037e90c4de184c0f0be2d1df78d to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/ba4b6037e90c4de184c0f0be2d1df78d 2024-11-10T22:38:05,949 DEBUG [StoreCloser-TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/7f6b8c49516c43ed8259876beab13e46 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/7f6b8c49516c43ed8259876beab13e46 2024-11-10T22:38:05,950 DEBUG [StoreCloser-TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/7c2bc1b4217140ebabd5eb401f1075ea to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/7c2bc1b4217140ebabd5eb401f1075ea 2024-11-10T22:38:05,951 DEBUG [StoreCloser-TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/c150439f212f4834afc76f7dc510ee6b to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/c150439f212f4834afc76f7dc510ee6b 2024-11-10T22:38:05,952 DEBUG [StoreCloser-TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/dbadc1b2c799480b911a9d5c3c8c5044 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/dbadc1b2c799480b911a9d5c3c8c5044 2024-11-10T22:38:05,953 DEBUG [StoreCloser-TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/02dffc988c654acdb7c5b6636b1f8fb0 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/02dffc988c654acdb7c5b6636b1f8fb0 2024-11-10T22:38:05,954 DEBUG [StoreCloser-TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/2d8d0d7d82954738a297b75d445597be to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/2d8d0d7d82954738a297b75d445597be 2024-11-10T22:38:05,955 DEBUG [StoreCloser-TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/41a3911cc8004669b1827020c2992317 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/41a3911cc8004669b1827020c2992317 2024-11-10T22:38:05,956 DEBUG [StoreCloser-TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/b895e2363bde4657b03f95790ca56414 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/b895e2363bde4657b03f95790ca56414 2024-11-10T22:38:05,957 DEBUG [StoreCloser-TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/3ccf43dd083747d5b1a8f0ff6607ff79 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/3ccf43dd083747d5b1a8f0ff6607ff79 2024-11-10T22:38:05,958 DEBUG [StoreCloser-TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/6ce20a8e7b37485d9f4316f87fe029fb to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/6ce20a8e7b37485d9f4316f87fe029fb 2024-11-10T22:38:05,959 DEBUG [StoreCloser-TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/57f2f4f1eba44484a47acaf13112c371 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/57f2f4f1eba44484a47acaf13112c371 2024-11-10T22:38:05,960 DEBUG [StoreCloser-TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/2c424e9f1614487fbebe8b57481cdb54 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/2c424e9f1614487fbebe8b57481cdb54 2024-11-10T22:38:05,961 DEBUG [StoreCloser-TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/f533c10058cf43c1996bdbfba5f17afe to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/f533c10058cf43c1996bdbfba5f17afe 2024-11-10T22:38:05,962 DEBUG [StoreCloser-TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/6c10a4657cc14291b13fe1750b2dab2f to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/6c10a4657cc14291b13fe1750b2dab2f 2024-11-10T22:38:05,963 DEBUG [StoreCloser-TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/059ad6464df0438eb0b2cd7d679552c4 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/059ad6464df0438eb0b2cd7d679552c4 2024-11-10T22:38:05,964 DEBUG [StoreCloser-TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/06eed5fa99724f70af604cb7a95d7903 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/06eed5fa99724f70af604cb7a95d7903 2024-11-10T22:38:05,965 DEBUG [StoreCloser-TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/d26787b8090244f0a08df9f661a0ec44 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/d26787b8090244f0a08df9f661a0ec44 2024-11-10T22:38:05,966 DEBUG [StoreCloser-TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/c2e31da1398a4659a956fd2ae75b377a to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/c2e31da1398a4659a956fd2ae75b377a 2024-11-10T22:38:05,967 DEBUG [StoreCloser-TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/46b1c6d9a43b48c39f86135575a61a3e to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/46b1c6d9a43b48c39f86135575a61a3e 2024-11-10T22:38:05,972 DEBUG [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/recovered.edits/567.seqid, newMaxSeqId=567, maxSeqId=1 2024-11-10T22:38:05,973 INFO [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. 2024-11-10T22:38:05,973 DEBUG [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1676): Region close journal for dbbcde27e8d425fcc5932f5841a8c339: Waiting for close lock at 1731278284558Running coprocessor pre-close hooks at 1731278284558Disabling compacts and flushes for region at 1731278284558Disabling writes for close at 1731278284558Obtaining lock to block concurrent updates at 1731278284559 (+1 ms)Preparing flush snapshotting stores in dbbcde27e8d425fcc5932f5841a8c339 at 1731278284559Finished memstore snapshotting TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339., syncing WAL and waiting on mvcc, flushsize=dataSize=13740, getHeapSize=36720, getOffHeapSize=0, getCellsCount=300 at 1731278284559Flushing stores of TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339. at 1731278284559Flushing dbbcde27e8d425fcc5932f5841a8c339/A: creating writer at 1731278284560 (+1 ms)Flushing dbbcde27e8d425fcc5932f5841a8c339/A: appending metadata at 1731278284563 (+3 ms)Flushing dbbcde27e8d425fcc5932f5841a8c339/A: closing flushed file at 1731278284563Flushing dbbcde27e8d425fcc5932f5841a8c339/B: creating writer at 1731278284973 (+410 ms)Flushing dbbcde27e8d425fcc5932f5841a8c339/B: appending metadata at 1731278284975 (+2 ms)Flushing dbbcde27e8d425fcc5932f5841a8c339/B: closing flushed file at 1731278284975Flushing dbbcde27e8d425fcc5932f5841a8c339/C: creating writer at 1731278285384 (+409 ms)Flushing dbbcde27e8d425fcc5932f5841a8c339/C: appending metadata at 1731278285387 (+3 ms)Flushing dbbcde27e8d425fcc5932f5841a8c339/C: closing flushed file at 1731278285387Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7939b1ec: reopening flushed file at 1731278285796 (+409 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@446accc7: reopening flushed file at 1731278285801 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@26c865fc: reopening flushed file at 1731278285805 (+4 ms)Finished flush of dataSize ~13.42 KB/13740, heapSize ~35.86 KB/36720, currentSize=0 B/0 for dbbcde27e8d425fcc5932f5841a8c339 in 1252ms, sequenceid=564, compaction requested=false at 1731278285810 (+5 ms)Writing region close event to WAL at 1731278285969 (+159 ms)Running coprocessor post-close hooks at 1731278285972 (+3 ms)Closed at 1731278285973 (+1 ms) 2024-11-10T22:38:05,975 INFO [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] handler.UnassignRegionHandler(157): Closed dbbcde27e8d425fcc5932f5841a8c339 2024-11-10T22:38:05,976 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=dbbcde27e8d425fcc5932f5841a8c339, regionState=CLOSED 2024-11-10T22:38:05,978 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=33, state=RUNNABLE, hasLock=false; CloseRegionProcedure dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 because future has completed 2024-11-10T22:38:05,980 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=33 2024-11-10T22:38:05,980 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure dbbcde27e8d425fcc5932f5841a8c339, server=849d0d71310f,43753,1731278250663 in 1.5780 sec 2024-11-10T22:38:05,982 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=32 2024-11-10T22:38:05,982 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=dbbcde27e8d425fcc5932f5841a8c339, UNASSIGN in 1.5840 sec 2024-11-10T22:38:05,984 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-11-10T22:38:05,985 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5910 sec 2024-11-10T22:38:05,986 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731278285986"}]},"ts":"1731278285986"} 2024-11-10T22:38:05,987 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-10T22:38:05,988 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-10T22:38:05,989 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.6060 sec 2024-11-10T22:38:06,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-10T22:38:06,533 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-10T22:38:06,536 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-10T22:38:06,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] procedure2.ProcedureExecutor(1139): Stored pid=35, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T22:38:06,541 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=35, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T22:38:06,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=35 2024-11-10T22:38:06,543 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=35, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T22:38:06,547 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339 2024-11-10T22:38:06,551 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A, FileablePath, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B, FileablePath, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C, FileablePath, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/recovered.edits] 2024-11-10T22:38:06,553 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/1f88a6717d8246cea5dc89b7ece4eb03 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/1f88a6717d8246cea5dc89b7ece4eb03 2024-11-10T22:38:06,555 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/7594984ef5974a20a2a5f771902ac65c to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/A/7594984ef5974a20a2a5f771902ac65c 2024-11-10T22:38:06,557 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/1aec3888fbf4467883d12854e34a360c to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/1aec3888fbf4467883d12854e34a360c 2024-11-10T22:38:06,558 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/c15eb60719d448078476c2b8bb0b5c73 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/B/c15eb60719d448078476c2b8bb0b5c73 2024-11-10T22:38:06,560 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/b85a88b1ae96468fab19983605271202 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/b85a88b1ae96468fab19983605271202 2024-11-10T22:38:06,562 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/eaeaf8fcffe04830888ef5b4f46b13cf to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/C/eaeaf8fcffe04830888ef5b4f46b13cf 2024-11-10T22:38:06,564 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/recovered.edits/567.seqid to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339/recovered.edits/567.seqid 2024-11-10T22:38:06,565 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/dbbcde27e8d425fcc5932f5841a8c339 2024-11-10T22:38:06,565 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-10T22:38:06,570 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=35, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T22:38:06,575 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43753 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-11-10T22:38:06,579 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-10T22:38:06,609 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-10T22:38:06,611 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=35, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T22:38:06,611 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-10T22:38:06,611 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731278286611"}]},"ts":"9223372036854775807"} 2024-11-10T22:38:06,615 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-10T22:38:06,615 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => dbbcde27e8d425fcc5932f5841a8c339, NAME => 'TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339.', STARTKEY => '', ENDKEY => ''}] 2024-11-10T22:38:06,616 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-10T22:38:06,616 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731278286616"}]},"ts":"9223372036854775807"} 2024-11-10T22:38:06,618 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-10T22:38:06,619 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=35, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T22:38:06,620 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 82 msec 2024-11-10T22:38:06,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=35 2024-11-10T22:38:06,653 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-10T22:38:06,654 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-10T22:38:06,667 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMixedAtomicity Thread=237 (was 211) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/849d0d71310f:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;849d0d71310f:43753-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-196171798_22 at /127.0.0.1:44096 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=455 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=161 (was 39) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=7207 (was 7735) 2024-11-10T22:38:06,677 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobMixedAtomicity Thread=237, OpenFileDescriptor=455, MaxFileDescriptor=1048576, SystemLoadAverage=161, ProcessCount=11, AvailableMemoryMB=7205 2024-11-10T22:38:06,679 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-10T22:38:06,679 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-10T22:38:06,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] procedure2.ProcedureExecutor(1139): Stored pid=36, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-10T22:38:06,681 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=36, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-10T22:38:06,681 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:06,682 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 36 2024-11-10T22:38:06,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-10T22:38:06,682 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=36, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-10T22:38:06,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073741960_1136 (size=959) 2024-11-10T22:38:06,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-10T22:38:07,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-10T22:38:07,091 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => d51b53151407a9b9ad82a60ad302f998, NAME => 'TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39 2024-11-10T22:38:07,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073741961_1137 (size=53) 2024-11-10T22:38:07,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-10T22:38:07,497 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T22:38:07,497 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing d51b53151407a9b9ad82a60ad302f998, disabling compactions & flushes 2024-11-10T22:38:07,497 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. 2024-11-10T22:38:07,497 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. 2024-11-10T22:38:07,497 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. after waiting 0 ms 2024-11-10T22:38:07,497 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. 2024-11-10T22:38:07,497 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. 2024-11-10T22:38:07,497 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for d51b53151407a9b9ad82a60ad302f998: Waiting for close lock at 1731278287497Disabling compacts and flushes for region at 1731278287497Disabling writes for close at 1731278287497Writing region close event to WAL at 1731278287497Closed at 1731278287497 2024-11-10T22:38:07,499 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=36, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-10T22:38:07,499 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731278287499"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731278287499"}]},"ts":"1731278287499"} 2024-11-10T22:38:07,501 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-10T22:38:07,501 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=36, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-10T22:38:07,502 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731278287501"}]},"ts":"1731278287501"} 2024-11-10T22:38:07,503 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-10T22:38:07,503 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d51b53151407a9b9ad82a60ad302f998, ASSIGN}] 2024-11-10T22:38:07,505 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d51b53151407a9b9ad82a60ad302f998, ASSIGN 2024-11-10T22:38:07,505 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d51b53151407a9b9ad82a60ad302f998, ASSIGN; state=OFFLINE, location=849d0d71310f,43753,1731278250663; forceNewPlan=false, retain=false 2024-11-10T22:38:07,656 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=d51b53151407a9b9ad82a60ad302f998, regionState=OPENING, regionLocation=849d0d71310f,43753,1731278250663 2024-11-10T22:38:07,658 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d51b53151407a9b9ad82a60ad302f998, ASSIGN because future has completed 2024-11-10T22:38:07,659 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=37, state=RUNNABLE, hasLock=false; OpenRegionProcedure d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663}] 2024-11-10T22:38:07,814 INFO [RS_OPEN_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. 2024-11-10T22:38:07,814 DEBUG [RS_OPEN_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7752): Opening region: {ENCODED => d51b53151407a9b9ad82a60ad302f998, NAME => 'TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.', STARTKEY => '', ENDKEY => ''} 2024-11-10T22:38:07,815 DEBUG [RS_OPEN_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees d51b53151407a9b9ad82a60ad302f998 2024-11-10T22:38:07,815 DEBUG [RS_OPEN_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T22:38:07,815 DEBUG [RS_OPEN_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7794): checking encryption for d51b53151407a9b9ad82a60ad302f998 2024-11-10T22:38:07,815 DEBUG [RS_OPEN_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7797): checking classloading for d51b53151407a9b9ad82a60ad302f998 2024-11-10T22:38:07,816 INFO [StoreOpener-d51b53151407a9b9ad82a60ad302f998-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region d51b53151407a9b9ad82a60ad302f998 2024-11-10T22:38:07,818 INFO [StoreOpener-d51b53151407a9b9ad82a60ad302f998-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d51b53151407a9b9ad82a60ad302f998 columnFamilyName A 2024-11-10T22:38:07,818 DEBUG [StoreOpener-d51b53151407a9b9ad82a60ad302f998-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:07,819 INFO [StoreOpener-d51b53151407a9b9ad82a60ad302f998-1 {}] regionserver.HStore(327): Store=d51b53151407a9b9ad82a60ad302f998/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T22:38:07,819 INFO [StoreOpener-d51b53151407a9b9ad82a60ad302f998-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region d51b53151407a9b9ad82a60ad302f998 2024-11-10T22:38:07,820 INFO [StoreOpener-d51b53151407a9b9ad82a60ad302f998-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d51b53151407a9b9ad82a60ad302f998 columnFamilyName B 2024-11-10T22:38:07,820 DEBUG [StoreOpener-d51b53151407a9b9ad82a60ad302f998-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:07,821 INFO [StoreOpener-d51b53151407a9b9ad82a60ad302f998-1 {}] regionserver.HStore(327): Store=d51b53151407a9b9ad82a60ad302f998/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T22:38:07,821 INFO [StoreOpener-d51b53151407a9b9ad82a60ad302f998-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region d51b53151407a9b9ad82a60ad302f998 2024-11-10T22:38:07,822 INFO [StoreOpener-d51b53151407a9b9ad82a60ad302f998-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d51b53151407a9b9ad82a60ad302f998 columnFamilyName C 2024-11-10T22:38:07,822 DEBUG [StoreOpener-d51b53151407a9b9ad82a60ad302f998-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:07,822 INFO [StoreOpener-d51b53151407a9b9ad82a60ad302f998-1 {}] regionserver.HStore(327): Store=d51b53151407a9b9ad82a60ad302f998/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T22:38:07,822 DEBUG [RS_OPEN_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1038): replaying wal for d51b53151407a9b9ad82a60ad302f998 2024-11-10T22:38:07,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-10T22:38:07,823 DEBUG [RS_OPEN_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998 2024-11-10T22:38:07,823 DEBUG [RS_OPEN_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998 2024-11-10T22:38:07,824 DEBUG [RS_OPEN_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1048): stopping wal replay for d51b53151407a9b9ad82a60ad302f998 2024-11-10T22:38:07,824 DEBUG [RS_OPEN_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1060): Cleaning up temporary data for d51b53151407a9b9ad82a60ad302f998 2024-11-10T22:38:07,825 DEBUG [RS_OPEN_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-10T22:38:07,826 DEBUG [RS_OPEN_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1093): writing seq id for d51b53151407a9b9ad82a60ad302f998 2024-11-10T22:38:07,828 DEBUG [RS_OPEN_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-10T22:38:07,828 INFO [RS_OPEN_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1114): Opened d51b53151407a9b9ad82a60ad302f998; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69651989, jitterRate=0.03789551556110382}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-10T22:38:07,828 DEBUG [RS_OPEN_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d51b53151407a9b9ad82a60ad302f998 2024-11-10T22:38:07,829 DEBUG [RS_OPEN_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1006): Region open journal for d51b53151407a9b9ad82a60ad302f998: Running coprocessor pre-open hook at 1731278287815Writing region info on filesystem at 1731278287815Initializing all the Stores at 1731278287816 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731278287816Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731278287816Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731278287816Cleaning up temporary data from old regions at 1731278287824 (+8 ms)Running coprocessor post-open hooks at 1731278287828 (+4 ms)Region opened successfully at 1731278287829 (+1 ms) 2024-11-10T22:38:07,829 INFO [RS_OPEN_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., pid=38, masterSystemTime=1731278287810 2024-11-10T22:38:07,831 DEBUG [RS_OPEN_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. 2024-11-10T22:38:07,831 INFO [RS_OPEN_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. 2024-11-10T22:38:07,832 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=d51b53151407a9b9ad82a60ad302f998, regionState=OPEN, openSeqNum=2, regionLocation=849d0d71310f,43753,1731278250663 2024-11-10T22:38:07,833 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=38, ppid=37, state=RUNNABLE, hasLock=false; OpenRegionProcedure d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 because future has completed 2024-11-10T22:38:07,837 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=38, resume processing ppid=37 2024-11-10T22:38:07,837 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, ppid=37, state=SUCCESS, hasLock=false; OpenRegionProcedure d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 in 176 msec 2024-11-10T22:38:07,838 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=36 2024-11-10T22:38:07,839 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=36, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d51b53151407a9b9ad82a60ad302f998, ASSIGN in 334 msec 2024-11-10T22:38:07,839 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=36, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-10T22:38:07,840 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731278287839"}]},"ts":"1731278287839"} 2024-11-10T22:38:07,841 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-10T22:38:07,842 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=36, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-10T22:38:07,844 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1630 sec 2024-11-10T22:38:08,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-10T22:38:08,833 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-10T22:38:08,834 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ffee9aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:38:08,834 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849d0d71310f,43351,-1 for getting cluster id 2024-11-10T22:38:08,834 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T22:38:08,835 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59ccc382-681c-4202-a6b6-e337b1afc6c7' 2024-11-10T22:38:08,836 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T22:38:08,836 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59ccc382-681c-4202-a6b6-e337b1afc6c7" 2024-11-10T22:38:08,836 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c376f61, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:38:08,836 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849d0d71310f,43351,-1] 2024-11-10T22:38:08,836 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T22:38:08,837 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:38:08,838 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43436, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T22:38:08,838 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a07e7ee, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:38:08,839 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T22:38:08,840 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849d0d71310f,43753,1731278250663, seqNum=-1] 2024-11-10T22:38:08,840 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T22:38:08,841 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51180, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T22:38:08,842 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-10T22:38:08,843 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is 849d0d71310f,43351,1731278249979 2024-11-10T22:38:08,843 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@61fade 2024-11-10T22:38:08,843 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-10T22:38:08,844 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43442, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-10T22:38:08,849 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-10T22:38:08,849 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-10T22:38:08,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] procedure2.ProcedureExecutor(1139): Stored pid=39, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-10T22:38:08,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073741962_1138 (size=995) 2024-11-10T22:38:09,269 DEBUG [PEWorker-5 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-10T22:38:09,269 INFO [PEWorker-5 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-10T22:38:09,273 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=40, ppid=39, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-10T22:38:09,280 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d51b53151407a9b9ad82a60ad302f998, REOPEN/MOVE}] 2024-11-10T22:38:09,280 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=41, ppid=40, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d51b53151407a9b9ad82a60ad302f998, REOPEN/MOVE 2024-11-10T22:38:09,281 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=41 updating hbase:meta row=d51b53151407a9b9ad82a60ad302f998, regionState=CLOSING, regionLocation=849d0d71310f,43753,1731278250663 2024-11-10T22:38:09,283 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43351 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=849d0d71310f,43753,1731278250663, table=TestAcidGuarantees, region=d51b53151407a9b9ad82a60ad302f998. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-10T22:38:09,283 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=41, ppid=40, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d51b53151407a9b9ad82a60ad302f998, REOPEN/MOVE because future has completed 2024-11-10T22:38:09,284 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-10T22:38:09,284 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=42, ppid=41, state=RUNNABLE, hasLock=false; CloseRegionProcedure d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663}] 2024-11-10T22:38:09,436 INFO [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=42}] handler.UnassignRegionHandler(122): Close d51b53151407a9b9ad82a60ad302f998 2024-11-10T22:38:09,436 DEBUG [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=42}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-10T22:38:09,436 DEBUG [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=42}] regionserver.HRegion(1722): Closing d51b53151407a9b9ad82a60ad302f998, disabling compactions & flushes 2024-11-10T22:38:09,436 INFO [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=42}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. 2024-11-10T22:38:09,436 DEBUG [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=42}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. 2024-11-10T22:38:09,436 DEBUG [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=42}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. after waiting 0 ms 2024-11-10T22:38:09,437 DEBUG [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=42}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. 2024-11-10T22:38:09,440 DEBUG [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=42}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-10T22:38:09,441 INFO [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=42}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. 2024-11-10T22:38:09,441 DEBUG [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=42}] regionserver.HRegion(1676): Region close journal for d51b53151407a9b9ad82a60ad302f998: Waiting for close lock at 1731278289436Running coprocessor pre-close hooks at 1731278289436Disabling compacts and flushes for region at 1731278289436Disabling writes for close at 1731278289437 (+1 ms)Writing region close event to WAL at 1731278289437Running coprocessor post-close hooks at 1731278289441 (+4 ms)Closed at 1731278289441 2024-11-10T22:38:09,441 WARN [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=42}] regionserver.HRegionServer(3299): Not adding moved region record: d51b53151407a9b9ad82a60ad302f998 to self. 2024-11-10T22:38:09,443 INFO [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=42}] handler.UnassignRegionHandler(157): Closed d51b53151407a9b9ad82a60ad302f998 2024-11-10T22:38:09,444 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=41 updating hbase:meta row=d51b53151407a9b9ad82a60ad302f998, regionState=CLOSED 2024-11-10T22:38:09,445 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=42, ppid=41, state=RUNNABLE, hasLock=false; CloseRegionProcedure d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 because future has completed 2024-11-10T22:38:09,447 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=42, resume processing ppid=41 2024-11-10T22:38:09,447 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, ppid=41, state=SUCCESS, hasLock=false; CloseRegionProcedure d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 in 162 msec 2024-11-10T22:38:09,448 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=41, ppid=40, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d51b53151407a9b9ad82a60ad302f998, REOPEN/MOVE; state=CLOSED, location=849d0d71310f,43753,1731278250663; forceNewPlan=false, retain=true 2024-11-10T22:38:09,599 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=41 updating hbase:meta row=d51b53151407a9b9ad82a60ad302f998, regionState=OPENING, regionLocation=849d0d71310f,43753,1731278250663 2024-11-10T22:38:09,601 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=41, ppid=40, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d51b53151407a9b9ad82a60ad302f998, REOPEN/MOVE because future has completed 2024-11-10T22:38:09,601 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=41, state=RUNNABLE, hasLock=false; OpenRegionProcedure d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663}] 2024-11-10T22:38:09,756 INFO [RS_OPEN_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. 2024-11-10T22:38:09,757 DEBUG [RS_OPEN_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7752): Opening region: {ENCODED => d51b53151407a9b9ad82a60ad302f998, NAME => 'TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.', STARTKEY => '', ENDKEY => ''} 2024-11-10T22:38:09,757 DEBUG [RS_OPEN_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees d51b53151407a9b9ad82a60ad302f998 2024-11-10T22:38:09,757 DEBUG [RS_OPEN_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T22:38:09,757 DEBUG [RS_OPEN_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7794): checking encryption for d51b53151407a9b9ad82a60ad302f998 2024-11-10T22:38:09,757 DEBUG [RS_OPEN_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7797): checking classloading for d51b53151407a9b9ad82a60ad302f998 2024-11-10T22:38:09,760 INFO [StoreOpener-d51b53151407a9b9ad82a60ad302f998-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region d51b53151407a9b9ad82a60ad302f998 2024-11-10T22:38:09,765 INFO [StoreOpener-d51b53151407a9b9ad82a60ad302f998-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d51b53151407a9b9ad82a60ad302f998 columnFamilyName A 2024-11-10T22:38:09,767 DEBUG [StoreOpener-d51b53151407a9b9ad82a60ad302f998-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:09,768 INFO [StoreOpener-d51b53151407a9b9ad82a60ad302f998-1 {}] regionserver.HStore(327): Store=d51b53151407a9b9ad82a60ad302f998/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T22:38:09,768 INFO [StoreOpener-d51b53151407a9b9ad82a60ad302f998-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region d51b53151407a9b9ad82a60ad302f998 2024-11-10T22:38:09,769 INFO [StoreOpener-d51b53151407a9b9ad82a60ad302f998-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d51b53151407a9b9ad82a60ad302f998 columnFamilyName B 2024-11-10T22:38:09,769 DEBUG [StoreOpener-d51b53151407a9b9ad82a60ad302f998-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:09,769 INFO [StoreOpener-d51b53151407a9b9ad82a60ad302f998-1 {}] regionserver.HStore(327): Store=d51b53151407a9b9ad82a60ad302f998/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T22:38:09,769 INFO [StoreOpener-d51b53151407a9b9ad82a60ad302f998-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region d51b53151407a9b9ad82a60ad302f998 2024-11-10T22:38:09,770 INFO [StoreOpener-d51b53151407a9b9ad82a60ad302f998-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d51b53151407a9b9ad82a60ad302f998 columnFamilyName C 2024-11-10T22:38:09,770 DEBUG [StoreOpener-d51b53151407a9b9ad82a60ad302f998-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:09,770 INFO [StoreOpener-d51b53151407a9b9ad82a60ad302f998-1 {}] regionserver.HStore(327): Store=d51b53151407a9b9ad82a60ad302f998/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T22:38:09,771 DEBUG [RS_OPEN_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1038): replaying wal for d51b53151407a9b9ad82a60ad302f998 2024-11-10T22:38:09,771 DEBUG [RS_OPEN_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998 2024-11-10T22:38:09,772 DEBUG [RS_OPEN_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998 2024-11-10T22:38:09,773 DEBUG [RS_OPEN_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1048): stopping wal replay for d51b53151407a9b9ad82a60ad302f998 2024-11-10T22:38:09,773 DEBUG [RS_OPEN_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1060): Cleaning up temporary data for d51b53151407a9b9ad82a60ad302f998 2024-11-10T22:38:09,774 DEBUG [RS_OPEN_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-10T22:38:09,775 DEBUG [RS_OPEN_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1093): writing seq id for d51b53151407a9b9ad82a60ad302f998 2024-11-10T22:38:09,776 INFO [RS_OPEN_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1114): Opened d51b53151407a9b9ad82a60ad302f998; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66174408, jitterRate=-0.013924479484558105}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-10T22:38:09,776 DEBUG [RS_OPEN_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d51b53151407a9b9ad82a60ad302f998 2024-11-10T22:38:09,778 DEBUG [RS_OPEN_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1006): Region open journal for d51b53151407a9b9ad82a60ad302f998: Running coprocessor pre-open hook at 1731278289757Writing region info on filesystem at 1731278289757Initializing all the Stores at 1731278289758 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731278289758Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731278289759 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731278289759Cleaning up temporary data from old regions at 1731278289773 (+14 ms)Running coprocessor post-open hooks at 1731278289776 (+3 ms)Region opened successfully at 1731278289778 (+2 ms) 2024-11-10T22:38:09,779 INFO [RS_OPEN_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., pid=43, masterSystemTime=1731278289753 2024-11-10T22:38:09,781 DEBUG [RS_OPEN_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. 2024-11-10T22:38:09,781 INFO [RS_OPEN_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. 2024-11-10T22:38:09,781 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=41 updating hbase:meta row=d51b53151407a9b9ad82a60ad302f998, regionState=OPEN, openSeqNum=5, regionLocation=849d0d71310f,43753,1731278250663 2024-11-10T22:38:09,783 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=43, ppid=41, state=RUNNABLE, hasLock=false; OpenRegionProcedure d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 because future has completed 2024-11-10T22:38:09,785 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=41 2024-11-10T22:38:09,785 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=41, state=SUCCESS, hasLock=false; OpenRegionProcedure d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 in 183 msec 2024-11-10T22:38:09,787 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=40 2024-11-10T22:38:09,787 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=40, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d51b53151407a9b9ad82a60ad302f998, REOPEN/MOVE in 505 msec 2024-11-10T22:38:09,790 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=40, resume processing ppid=39 2024-11-10T22:38:09,790 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, ppid=39, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 516 msec 2024-11-10T22:38:09,794 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 940 msec 2024-11-10T22:38:09,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=39 2024-11-10T22:38:09,796 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-10T22:38:09,797 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f683619, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:38:09,797 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849d0d71310f,43351,-1 for getting cluster id 2024-11-10T22:38:09,797 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T22:38:09,798 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59ccc382-681c-4202-a6b6-e337b1afc6c7' 2024-11-10T22:38:09,798 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T22:38:09,798 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59ccc382-681c-4202-a6b6-e337b1afc6c7" 2024-11-10T22:38:09,798 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7eb553db, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:38:09,798 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849d0d71310f,43351,-1] 2024-11-10T22:38:09,799 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T22:38:09,799 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:38:09,800 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36958, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T22:38:09,800 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3293988f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:38:09,801 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@dd47c94, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:38:09,801 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849d0d71310f,43351,-1 for getting cluster id 2024-11-10T22:38:09,801 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T22:38:09,802 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59ccc382-681c-4202-a6b6-e337b1afc6c7' 2024-11-10T22:38:09,802 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T22:38:09,802 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59ccc382-681c-4202-a6b6-e337b1afc6c7" 2024-11-10T22:38:09,802 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@594dbf65, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:38:09,802 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849d0d71310f,43351,-1] 2024-11-10T22:38:09,802 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T22:38:09,802 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:38:09,803 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36976, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T22:38:09,804 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a4275f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:38:09,805 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@750c1845, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:38:09,805 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849d0d71310f,43351,-1 for getting cluster id 2024-11-10T22:38:09,805 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T22:38:09,805 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59ccc382-681c-4202-a6b6-e337b1afc6c7' 2024-11-10T22:38:09,805 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T22:38:09,806 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59ccc382-681c-4202-a6b6-e337b1afc6c7" 2024-11-10T22:38:09,806 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@44f8f8ef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:38:09,806 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849d0d71310f,43351,-1] 2024-11-10T22:38:09,806 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T22:38:09,806 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:38:09,807 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37000, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T22:38:09,807 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@735c15dd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:38:09,808 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@85a5f7a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:38:09,808 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849d0d71310f,43351,-1 for getting cluster id 2024-11-10T22:38:09,808 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T22:38:09,809 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59ccc382-681c-4202-a6b6-e337b1afc6c7' 2024-11-10T22:38:09,809 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T22:38:09,809 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59ccc382-681c-4202-a6b6-e337b1afc6c7" 2024-11-10T22:38:09,809 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@547a67e4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:38:09,809 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849d0d71310f,43351,-1] 2024-11-10T22:38:09,809 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T22:38:09,809 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:38:09,810 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37020, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T22:38:09,810 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f75878f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:38:09,811 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71151d6d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:38:09,811 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849d0d71310f,43351,-1 for getting cluster id 2024-11-10T22:38:09,811 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T22:38:09,812 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59ccc382-681c-4202-a6b6-e337b1afc6c7' 2024-11-10T22:38:09,812 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T22:38:09,812 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59ccc382-681c-4202-a6b6-e337b1afc6c7" 2024-11-10T22:38:09,812 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14b9d0fb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:38:09,812 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849d0d71310f,43351,-1] 2024-11-10T22:38:09,812 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T22:38:09,813 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:38:09,813 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37042, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T22:38:09,814 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@111c3457, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:38:09,815 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ab1fb6e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:38:09,815 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849d0d71310f,43351,-1 for getting cluster id 2024-11-10T22:38:09,815 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T22:38:09,815 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59ccc382-681c-4202-a6b6-e337b1afc6c7' 2024-11-10T22:38:09,816 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T22:38:09,816 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59ccc382-681c-4202-a6b6-e337b1afc6c7" 2024-11-10T22:38:09,816 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@76567690, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:38:09,816 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849d0d71310f,43351,-1] 2024-11-10T22:38:09,816 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T22:38:09,816 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:38:09,817 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37070, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T22:38:09,817 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6328c91a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:38:09,818 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6dd18334, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:38:09,818 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849d0d71310f,43351,-1 for getting cluster id 2024-11-10T22:38:09,818 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T22:38:09,819 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59ccc382-681c-4202-a6b6-e337b1afc6c7' 2024-11-10T22:38:09,819 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T22:38:09,819 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59ccc382-681c-4202-a6b6-e337b1afc6c7" 2024-11-10T22:38:09,819 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ddb9398, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:38:09,819 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849d0d71310f,43351,-1] 2024-11-10T22:38:09,819 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T22:38:09,819 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:38:09,820 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37096, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T22:38:09,820 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d887f81, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:38:09,821 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52c57d01, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:38:09,821 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849d0d71310f,43351,-1 for getting cluster id 2024-11-10T22:38:09,821 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T22:38:09,822 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59ccc382-681c-4202-a6b6-e337b1afc6c7' 2024-11-10T22:38:09,822 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T22:38:09,822 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59ccc382-681c-4202-a6b6-e337b1afc6c7" 2024-11-10T22:38:09,822 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63c39a4b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:38:09,822 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849d0d71310f,43351,-1] 2024-11-10T22:38:09,822 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T22:38:09,822 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:38:09,823 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37110, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T22:38:09,823 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a3b61a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:38:09,824 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b465b92, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:38:09,824 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849d0d71310f,43351,-1 for getting cluster id 2024-11-10T22:38:09,824 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T22:38:09,825 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59ccc382-681c-4202-a6b6-e337b1afc6c7' 2024-11-10T22:38:09,825 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T22:38:09,825 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59ccc382-681c-4202-a6b6-e337b1afc6c7" 2024-11-10T22:38:09,825 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5951bf29, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:38:09,825 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849d0d71310f,43351,-1] 2024-11-10T22:38:09,825 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T22:38:09,825 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:38:09,826 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37118, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T22:38:09,826 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62d1f2da, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:38:09,827 DEBUG [Thread-696 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T22:38:09,827 DEBUG [Thread-702 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T22:38:09,827 DEBUG [Thread-689 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T22:38:09,828 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T22:38:09,828 DEBUG [Thread-687 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T22:38:09,828 DEBUG [Thread-700 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T22:38:09,828 DEBUG [Thread-698 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T22:38:09,829 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849d0d71310f,43753,1731278250663, seqNum=-1] 2024-11-10T22:38:09,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-11-10T22:38:09,829 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849d0d71310f,43753,1731278250663, seqNum=-1] 2024-11-10T22:38:09,829 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849d0d71310f,43753,1731278250663, seqNum=-1] 2024-11-10T22:38:09,829 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849d0d71310f,43753,1731278250663, seqNum=-1] 2024-11-10T22:38:09,829 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T22:38:09,829 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T22:38:09,830 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849d0d71310f,43753,1731278250663, seqNum=-1] 2024-11-10T22:38:09,830 DEBUG [Thread-693 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T22:38:09,830 DEBUG [Thread-691 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T22:38:09,830 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T22:38:09,830 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T22:38:09,831 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T22:38:09,831 DEBUG [Thread-685 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T22:38:09,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-10T22:38:09,832 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849d0d71310f,43753,1731278250663, seqNum=-1] 2024-11-10T22:38:09,832 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40738, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T22:38:09,832 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T22:38:09,832 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849d0d71310f,43753,1731278250663, seqNum=-1] 2024-11-10T22:38:09,832 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40736, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T22:38:09,832 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T22:38:09,832 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40748, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T22:38:09,832 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40744, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T22:38:09,833 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T22:38:09,833 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40752, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T22:38:09,833 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849d0d71310f,43753,1731278250663, seqNum=-1] 2024-11-10T22:38:09,833 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40754, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T22:38:09,833 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40768, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T22:38:09,834 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T22:38:09,834 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849d0d71310f,43753,1731278250663, seqNum=-1] 2024-11-10T22:38:09,834 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T22:38:09,834 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40774, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T22:38:09,835 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T22:38:09,835 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T22:38:09,835 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40778, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T22:38:09,837 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5] 2024-11-10T22:38:09,838 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5] 2024-11-10T22:38:09,840 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5] 2024-11-10T22:38:09,843 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5] 2024-11-10T22:38:09,844 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5] 2024-11-10T22:38:09,846 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5] 2024-11-10T22:38:09,847 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5] 2024-11-10T22:38:09,849 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5] 2024-11-10T22:38:09,851 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5] 2024-11-10T22:38:09,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(8855): Flush requested on d51b53151407a9b9ad82a60ad302f998 2024-11-10T22:38:09,854 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T22:38:09,854 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d51b53151407a9b9ad82a60ad302f998 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-10T22:38:09,910 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110120e549b1da84568907d4b41461ce23a_d51b53151407a9b9ad82a60ad302f998 is 50, key is test_row_0/A:col10/1731278289853/Put/seqid=0 2024-11-10T22:38:09,921 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:09,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40736 deadline: 1731278349915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:09,922 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:09,922 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:09,922 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:09,928 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:09,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40774 deadline: 1731278349917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:09,929 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:09,929 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:09,929 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:09,929 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:09,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40778 deadline: 1731278349917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:09,930 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:09,930 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:09,930 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:09,930 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:09,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40738 deadline: 1731278349919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:09,931 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:09,931 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:09,931 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:09,931 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:09,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40754 deadline: 1731278349920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:09,931 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:09,931 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:09,931 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:09,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073741963_1139 (size=12154) 2024-11-10T22:38:09,935 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:09,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-10T22:38:09,946 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110120e549b1da84568907d4b41461ce23a_d51b53151407a9b9ad82a60ad302f998 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110120e549b1da84568907d4b41461ce23a_d51b53151407a9b9ad82a60ad302f998 2024-11-10T22:38:09,947 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/A/cfcb2c04057f42798b3273500fe63c42, store: [table=TestAcidGuarantees family=A region=d51b53151407a9b9ad82a60ad302f998] 2024-11-10T22:38:09,957 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/A/cfcb2c04057f42798b3273500fe63c42 is 175, key is test_row_0/A:col10/1731278289853/Put/seqid=0 2024-11-10T22:38:09,972 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073741964_1140 (size=30955) 2024-11-10T22:38:09,988 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-10T22:38:09,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. 2024-11-10T22:38:09,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. as already flushing 2024-11-10T22:38:09,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. 2024-11-10T22:38:09,988 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => d51b53151407a9b9ad82a60ad302f998, NAME => 'TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:09,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => d51b53151407a9b9ad82a60ad302f998, NAME => 'TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:09,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d51b53151407a9b9ad82a60ad302f998, NAME => 'TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d51b53151407a9b9ad82a60ad302f998, NAME => 'TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:10,034 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:10,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40738 deadline: 1731278350033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:10,035 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:10,035 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:10,035 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:10,035 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:10,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40778 deadline: 1731278350034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:10,036 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:10,036 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:10,036 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:10,037 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:10,037 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:10,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40774 deadline: 1731278350034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:10,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40736 deadline: 1731278350034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:10,038 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:10,038 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:10,038 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:10,038 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:10,038 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:10,038 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:10,049 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:10,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40754 deadline: 1731278350043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:10,049 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:10,050 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:10,050 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:10,143 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-10T22:38:10,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. 2024-11-10T22:38:10,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. as already flushing 2024-11-10T22:38:10,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. 2024-11-10T22:38:10,143 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => d51b53151407a9b9ad82a60ad302f998, NAME => 'TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:10,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => d51b53151407a9b9ad82a60ad302f998, NAME => 'TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:10,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d51b53151407a9b9ad82a60ad302f998, NAME => 'TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d51b53151407a9b9ad82a60ad302f998, NAME => 'TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:10,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-10T22:38:10,243 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:10,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40738 deadline: 1731278350243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:10,244 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:10,244 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:10,244 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:10,244 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:10,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40774 deadline: 1731278350243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:10,244 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:10,244 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:10,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40736 deadline: 1731278350243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:10,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40778 deadline: 1731278350243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:10,245 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:10,245 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:10,245 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:10,246 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:10,246 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:10,246 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:10,246 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:10,246 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:10,246 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:10,255 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:10,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40754 deadline: 1731278350252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:10,256 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:10,256 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:10,256 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:10,298 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-10T22:38:10,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. 2024-11-10T22:38:10,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. as already flushing 2024-11-10T22:38:10,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. 2024-11-10T22:38:10,299 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => d51b53151407a9b9ad82a60ad302f998, NAME => 'TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:10,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => d51b53151407a9b9ad82a60ad302f998, NAME => 'TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:10,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d51b53151407a9b9ad82a60ad302f998, NAME => 'TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d51b53151407a9b9ad82a60ad302f998, NAME => 'TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:10,376 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/A/cfcb2c04057f42798b3273500fe63c42 2024-11-10T22:38:10,412 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/B/8629a25e9fb34d728400b1dd1d2f8f1d is 50, key is test_row_0/B:col10/1731278289853/Put/seqid=0 2024-11-10T22:38:10,416 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073741965_1141 (size=12001) 2024-11-10T22:38:10,418 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/B/8629a25e9fb34d728400b1dd1d2f8f1d 2024-11-10T22:38:10,452 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-10T22:38:10,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. 2024-11-10T22:38:10,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. as already flushing 2024-11-10T22:38:10,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. 2024-11-10T22:38:10,453 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => d51b53151407a9b9ad82a60ad302f998, NAME => 'TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:10,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => d51b53151407a9b9ad82a60ad302f998, NAME => 'TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:10,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d51b53151407a9b9ad82a60ad302f998, NAME => 'TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d51b53151407a9b9ad82a60ad302f998, NAME => 'TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:10,459 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/C/fcfc690455ef47aca9ba62bf8c8e56a0 is 50, key is test_row_0/C:col10/1731278289853/Put/seqid=0 2024-11-10T22:38:10,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-10T22:38:10,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073741966_1142 (size=12001) 2024-11-10T22:38:10,480 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/C/fcfc690455ef47aca9ba62bf8c8e56a0 2024-11-10T22:38:10,487 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/A/cfcb2c04057f42798b3273500fe63c42 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/cfcb2c04057f42798b3273500fe63c42 2024-11-10T22:38:10,496 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/cfcb2c04057f42798b3273500fe63c42, entries=150, sequenceid=15, filesize=30.2 K 2024-11-10T22:38:10,497 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/B/8629a25e9fb34d728400b1dd1d2f8f1d as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B/8629a25e9fb34d728400b1dd1d2f8f1d 2024-11-10T22:38:10,505 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B/8629a25e9fb34d728400b1dd1d2f8f1d, entries=150, sequenceid=15, filesize=11.7 K 2024-11-10T22:38:10,507 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/C/fcfc690455ef47aca9ba62bf8c8e56a0 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/C/fcfc690455ef47aca9ba62bf8c8e56a0 2024-11-10T22:38:10,514 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/C/fcfc690455ef47aca9ba62bf8c8e56a0, entries=150, sequenceid=15, filesize=11.7 K 2024-11-10T22:38:10,515 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for d51b53151407a9b9ad82a60ad302f998 in 661ms, sequenceid=15, compaction requested=false 2024-11-10T22:38:10,515 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d51b53151407a9b9ad82a60ad302f998: 2024-11-10T22:38:10,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(8855): Flush requested on d51b53151407a9b9ad82a60ad302f998 2024-11-10T22:38:10,558 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T22:38:10,558 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d51b53151407a9b9ad82a60ad302f998 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-10T22:38:10,575 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110c0b100caa185402682c0971674d90a98_d51b53151407a9b9ad82a60ad302f998 is 50, key is test_row_0/A:col10/1731278289898/Put/seqid=0 2024-11-10T22:38:10,579 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:10,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40774 deadline: 1731278350574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:10,580 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:10,580 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:10,580 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:10,580 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:10,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40738 deadline: 1731278350576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:10,581 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:10,581 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:10,581 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:10,582 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:10,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40754 deadline: 1731278350578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:10,583 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:10,583 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:10,583 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:10,583 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:10,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40736 deadline: 1731278350580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:10,584 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:10,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40778 deadline: 1731278350580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:10,585 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:10,585 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:10,585 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:10,585 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:10,585 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:10,585 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:10,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073741967_1143 (size=14594) 2024-11-10T22:38:10,596 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:10,603 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110c0b100caa185402682c0971674d90a98_d51b53151407a9b9ad82a60ad302f998 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110c0b100caa185402682c0971674d90a98_d51b53151407a9b9ad82a60ad302f998 2024-11-10T22:38:10,605 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/A/85b308ab4be1446892c81e8011f69e87, store: [table=TestAcidGuarantees family=A region=d51b53151407a9b9ad82a60ad302f998] 2024-11-10T22:38:10,605 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/A/85b308ab4be1446892c81e8011f69e87 is 175, key is test_row_0/A:col10/1731278289898/Put/seqid=0 2024-11-10T22:38:10,608 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-10T22:38:10,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. 2024-11-10T22:38:10,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. as already flushing 2024-11-10T22:38:10,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. 2024-11-10T22:38:10,608 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => d51b53151407a9b9ad82a60ad302f998, NAME => 'TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:10,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => d51b53151407a9b9ad82a60ad302f998, NAME => 'TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:10,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d51b53151407a9b9ad82a60ad302f998, NAME => 'TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d51b53151407a9b9ad82a60ad302f998, NAME => 'TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:10,611 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43351 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=849d0d71310f,43753,1731278250663, table=TestAcidGuarantees, region=d51b53151407a9b9ad82a60ad302f998. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-10T22:38:10,625 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073741968_1144 (size=39549) 2024-11-10T22:38:10,688 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:10,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40774 deadline: 1731278350682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:10,688 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:10,689 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:10,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40738 deadline: 1731278350683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:10,689 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:10,689 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:10,689 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:10,689 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:10,689 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:10,694 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:10,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40736 deadline: 1731278350693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:10,697 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:10,697 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:10,697 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:10,700 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:10,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40778 deadline: 1731278350695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:10,701 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:10,701 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:10,701 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:10,701 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:10,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40754 deadline: 1731278350695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:10,702 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:10,702 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:10,702 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:10,763 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-10T22:38:10,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. 2024-11-10T22:38:10,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. as already flushing 2024-11-10T22:38:10,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. 2024-11-10T22:38:10,763 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => d51b53151407a9b9ad82a60ad302f998, NAME => 'TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:10,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => d51b53151407a9b9ad82a60ad302f998, NAME => 'TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:10,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d51b53151407a9b9ad82a60ad302f998, NAME => 'TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d51b53151407a9b9ad82a60ad302f998, NAME => 'TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:10,893 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:10,893 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:10,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40774 deadline: 1731278350893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:10,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40738 deadline: 1731278350893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:10,894 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:10,894 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:10,894 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:10,894 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:10,894 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:10,894 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:10,904 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:10,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40736 deadline: 1731278350904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:10,905 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:10,905 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:10,905 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:10,917 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-10T22:38:10,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. 2024-11-10T22:38:10,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. as already flushing 2024-11-10T22:38:10,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. 2024-11-10T22:38:10,918 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => d51b53151407a9b9ad82a60ad302f998, NAME => 'TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:10,918 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:10,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => d51b53151407a9b9ad82a60ad302f998, NAME => 'TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:10,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40778 deadline: 1731278350912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:10,918 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:10,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40754 deadline: 1731278350913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:10,919 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:10,919 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:10,919 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:10,919 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:10,919 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:10,919 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:10,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d51b53151407a9b9ad82a60ad302f998, NAME => 'TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d51b53151407a9b9ad82a60ad302f998, NAME => 'TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:10,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-10T22:38:11,026 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=41, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/A/85b308ab4be1446892c81e8011f69e87 2024-11-10T22:38:11,045 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/B/a0127e246772431f8a0317025a507e44 is 50, key is test_row_0/B:col10/1731278289898/Put/seqid=0 2024-11-10T22:38:11,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073741969_1145 (size=12001) 2024-11-10T22:38:11,062 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/B/a0127e246772431f8a0317025a507e44 2024-11-10T22:38:11,073 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-10T22:38:11,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. 2024-11-10T22:38:11,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. as already flushing 2024-11-10T22:38:11,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. 2024-11-10T22:38:11,073 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => d51b53151407a9b9ad82a60ad302f998, NAME => 'TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:11,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => d51b53151407a9b9ad82a60ad302f998, NAME => 'TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:11,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d51b53151407a9b9ad82a60ad302f998, NAME => 'TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d51b53151407a9b9ad82a60ad302f998, NAME => 'TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:11,081 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/C/fa96840e58464ec8b93c24eeb453d3e8 is 50, key is test_row_0/C:col10/1731278289898/Put/seqid=0 2024-11-10T22:38:11,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073741970_1146 (size=12001) 2024-11-10T22:38:11,100 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/C/fa96840e58464ec8b93c24eeb453d3e8 2024-11-10T22:38:11,110 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/A/85b308ab4be1446892c81e8011f69e87 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/85b308ab4be1446892c81e8011f69e87 2024-11-10T22:38:11,118 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/85b308ab4be1446892c81e8011f69e87, entries=200, sequenceid=41, filesize=38.6 K 2024-11-10T22:38:11,120 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/B/a0127e246772431f8a0317025a507e44 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B/a0127e246772431f8a0317025a507e44 2024-11-10T22:38:11,127 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B/a0127e246772431f8a0317025a507e44, entries=150, sequenceid=41, filesize=11.7 K 2024-11-10T22:38:11,129 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/C/fa96840e58464ec8b93c24eeb453d3e8 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/C/fa96840e58464ec8b93c24eeb453d3e8 2024-11-10T22:38:11,145 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/C/fa96840e58464ec8b93c24eeb453d3e8, entries=150, sequenceid=41, filesize=11.7 K 2024-11-10T22:38:11,146 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=53.67 KB/54960 for d51b53151407a9b9ad82a60ad302f998 in 588ms, sequenceid=41, compaction requested=false 2024-11-10T22:38:11,146 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d51b53151407a9b9ad82a60ad302f998: 2024-11-10T22:38:11,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(8855): Flush requested on d51b53151407a9b9ad82a60ad302f998 2024-11-10T22:38:11,207 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T22:38:11,207 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d51b53151407a9b9ad82a60ad302f998 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-10T22:38:11,235 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-10T22:38:11,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. 2024-11-10T22:38:11,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. as already flushing 2024-11-10T22:38:11,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. 2024-11-10T22:38:11,236 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => d51b53151407a9b9ad82a60ad302f998, NAME => 'TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:11,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => d51b53151407a9b9ad82a60ad302f998, NAME => 'TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:11,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d51b53151407a9b9ad82a60ad302f998, NAME => 'TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d51b53151407a9b9ad82a60ad302f998, NAME => 'TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:11,244 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110b5bf10eb3b28475381e3a597d733d2ac_d51b53151407a9b9ad82a60ad302f998 is 50, key is test_row_0/A:col10/1731278291204/Put/seqid=0 2024-11-10T22:38:11,248 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073741971_1147 (size=17034) 2024-11-10T22:38:11,249 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:11,254 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110b5bf10eb3b28475381e3a597d733d2ac_d51b53151407a9b9ad82a60ad302f998 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110b5bf10eb3b28475381e3a597d733d2ac_d51b53151407a9b9ad82a60ad302f998 2024-11-10T22:38:11,255 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/A/6a77243a66f447039c7e1c471eb3a4aa, store: [table=TestAcidGuarantees family=A region=d51b53151407a9b9ad82a60ad302f998] 2024-11-10T22:38:11,256 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/A/6a77243a66f447039c7e1c471eb3a4aa is 175, key is test_row_0/A:col10/1731278291204/Put/seqid=0 2024-11-10T22:38:11,265 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073741972_1148 (size=48139) 2024-11-10T22:38:11,309 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:11,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40754 deadline: 1731278351303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:11,310 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:11,310 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:11,310 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:11,319 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:11,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40736 deadline: 1731278351305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:11,320 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:11,320 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:11,320 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:11,320 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:11,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40778 deadline: 1731278351305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:11,320 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:11,320 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:11,320 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:11,320 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:11,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40774 deadline: 1731278351305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:11,321 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:11,321 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:11,321 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:11,327 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:11,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40738 deadline: 1731278351322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:11,327 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:11,327 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:11,327 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:11,390 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-10T22:38:11,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. 2024-11-10T22:38:11,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. as already flushing 2024-11-10T22:38:11,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. 2024-11-10T22:38:11,391 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => d51b53151407a9b9ad82a60ad302f998, NAME => 'TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:11,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => d51b53151407a9b9ad82a60ad302f998, NAME => 'TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:11,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d51b53151407a9b9ad82a60ad302f998, NAME => 'TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d51b53151407a9b9ad82a60ad302f998, NAME => 'TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:11,418 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:11,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40754 deadline: 1731278351413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:11,419 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:11,419 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:11,419 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:11,423 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:11,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40736 deadline: 1731278351422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:11,423 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:11,423 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:11,423 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:11,423 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:11,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40778 deadline: 1731278351422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:11,424 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:11,424 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:11,424 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:11,437 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:11,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40774 deadline: 1731278351433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:11,437 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:11,437 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:11,437 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:11,437 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:11,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40738 deadline: 1731278351433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:11,439 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:11,439 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:11,439 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:11,515 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-10T22:38:11,545 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-10T22:38:11,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. 2024-11-10T22:38:11,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. as already flushing 2024-11-10T22:38:11,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. 2024-11-10T22:38:11,546 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => d51b53151407a9b9ad82a60ad302f998, NAME => 'TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:11,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => d51b53151407a9b9ad82a60ad302f998, NAME => 'TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:11,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d51b53151407a9b9ad82a60ad302f998, NAME => 'TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d51b53151407a9b9ad82a60ad302f998, NAME => 'TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:11,627 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:11,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40754 deadline: 1731278351622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:11,628 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:11,628 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:11,628 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:11,636 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:11,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40778 deadline: 1731278351633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:11,637 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:11,637 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:11,637 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:11,641 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:11,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40736 deadline: 1731278351633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:11,641 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:11,641 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:11,641 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:11,644 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:11,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40774 deadline: 1731278351642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:11,644 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:11,644 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:11,644 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:11,644 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:11,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40738 deadline: 1731278351642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:11,645 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:11,645 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:11,645 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:11,666 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=54, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/A/6a77243a66f447039c7e1c471eb3a4aa 2024-11-10T22:38:11,676 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/B/8bc9bce4fe03423e839ff893e172c332 is 50, key is test_row_0/B:col10/1731278291204/Put/seqid=0 2024-11-10T22:38:11,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073741973_1149 (size=12001) 2024-11-10T22:38:11,700 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-10T22:38:11,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. 2024-11-10T22:38:11,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. as already flushing 2024-11-10T22:38:11,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. 2024-11-10T22:38:11,700 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => d51b53151407a9b9ad82a60ad302f998, NAME => 'TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:11,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => d51b53151407a9b9ad82a60ad302f998, NAME => 'TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:11,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d51b53151407a9b9ad82a60ad302f998, NAME => 'TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d51b53151407a9b9ad82a60ad302f998, NAME => 'TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:11,854 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-10T22:38:11,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. 2024-11-10T22:38:11,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. as already flushing 2024-11-10T22:38:11,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. 2024-11-10T22:38:11,854 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => d51b53151407a9b9ad82a60ad302f998, NAME => 'TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:11,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => d51b53151407a9b9ad82a60ad302f998, NAME => 'TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:11,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d51b53151407a9b9ad82a60ad302f998, NAME => 'TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d51b53151407a9b9ad82a60ad302f998, NAME => 'TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:11,938 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:11,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40754 deadline: 1731278351932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:11,938 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:11,938 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:11,938 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:11,944 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:11,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40778 deadline: 1731278351943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:11,944 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:11,944 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:11,944 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:11,953 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:11,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40736 deadline: 1731278351952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:11,953 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:11,953 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:11,953 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:11,954 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:11,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40774 deadline: 1731278351952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:11,954 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:11,954 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:11,954 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:11,954 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:11,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40738 deadline: 1731278351952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:11,954 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:11,954 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:11,954 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:11,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-10T22:38:12,007 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-10T22:38:12,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. 2024-11-10T22:38:12,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. as already flushing 2024-11-10T22:38:12,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. 2024-11-10T22:38:12,008 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => d51b53151407a9b9ad82a60ad302f998, NAME => 'TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:12,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => d51b53151407a9b9ad82a60ad302f998, NAME => 'TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:12,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d51b53151407a9b9ad82a60ad302f998, NAME => 'TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d51b53151407a9b9ad82a60ad302f998, NAME => 'TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:12,085 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=54 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/B/8bc9bce4fe03423e839ff893e172c332 2024-11-10T22:38:12,095 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/C/65aeccd962204855b671817c73e75948 is 50, key is test_row_0/C:col10/1731278291204/Put/seqid=0 2024-11-10T22:38:12,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073741974_1150 (size=12001) 2024-11-10T22:38:12,104 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=54 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/C/65aeccd962204855b671817c73e75948 2024-11-10T22:38:12,110 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/A/6a77243a66f447039c7e1c471eb3a4aa as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/6a77243a66f447039c7e1c471eb3a4aa 2024-11-10T22:38:12,118 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/6a77243a66f447039c7e1c471eb3a4aa, entries=250, sequenceid=54, filesize=47.0 K 2024-11-10T22:38:12,119 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/B/8bc9bce4fe03423e839ff893e172c332 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B/8bc9bce4fe03423e839ff893e172c332 2024-11-10T22:38:12,126 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B/8bc9bce4fe03423e839ff893e172c332, entries=150, sequenceid=54, filesize=11.7 K 2024-11-10T22:38:12,127 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/C/65aeccd962204855b671817c73e75948 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/C/65aeccd962204855b671817c73e75948 2024-11-10T22:38:12,137 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/C/65aeccd962204855b671817c73e75948, entries=150, sequenceid=54, filesize=11.7 K 2024-11-10T22:38:12,138 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=140.89 KB/144270 for d51b53151407a9b9ad82a60ad302f998 in 931ms, sequenceid=54, compaction requested=true 2024-11-10T22:38:12,138 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d51b53151407a9b9ad82a60ad302f998: 2024-11-10T22:38:12,138 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d51b53151407a9b9ad82a60ad302f998:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T22:38:12,139 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T22:38:12,139 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T22:38:12,139 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T22:38:12,139 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d51b53151407a9b9ad82a60ad302f998:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T22:38:12,139 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T22:38:12,139 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d51b53151407a9b9ad82a60ad302f998:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T22:38:12,139 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T22:38:12,140 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T22:38:12,140 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 118643 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T22:38:12,140 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1541): d51b53151407a9b9ad82a60ad302f998/B is initiating minor compaction (all files) 2024-11-10T22:38:12,140 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HStore(1541): d51b53151407a9b9ad82a60ad302f998/A is initiating minor compaction (all files) 2024-11-10T22:38:12,140 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d51b53151407a9b9ad82a60ad302f998/B in TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. 2024-11-10T22:38:12,140 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d51b53151407a9b9ad82a60ad302f998/A in TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. 2024-11-10T22:38:12,140 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B/8629a25e9fb34d728400b1dd1d2f8f1d, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B/a0127e246772431f8a0317025a507e44, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B/8bc9bce4fe03423e839ff893e172c332] into tmpdir=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp, totalSize=35.2 K 2024-11-10T22:38:12,140 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/cfcb2c04057f42798b3273500fe63c42, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/85b308ab4be1446892c81e8011f69e87, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/6a77243a66f447039c7e1c471eb3a4aa] into tmpdir=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp, totalSize=115.9 K 2024-11-10T22:38:12,141 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. 2024-11-10T22:38:12,141 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. files: [hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/cfcb2c04057f42798b3273500fe63c42, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/85b308ab4be1446892c81e8011f69e87, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/6a77243a66f447039c7e1c471eb3a4aa] 2024-11-10T22:38:12,142 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting 8629a25e9fb34d728400b1dd1d2f8f1d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731278289842 2024-11-10T22:38:12,142 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting cfcb2c04057f42798b3273500fe63c42, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731278289842 2024-11-10T22:38:12,142 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting a0127e246772431f8a0317025a507e44, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1731278289898 2024-11-10T22:38:12,144 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 85b308ab4be1446892c81e8011f69e87, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1731278289898 2024-11-10T22:38:12,145 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting 8bc9bce4fe03423e839ff893e172c332, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731278290577 2024-11-10T22:38:12,145 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6a77243a66f447039c7e1c471eb3a4aa, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731278290559 2024-11-10T22:38:12,161 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-10T22:38:12,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. 2024-11-10T22:38:12,161 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing d51b53151407a9b9ad82a60ad302f998 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-10T22:38:12,169 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d51b53151407a9b9ad82a60ad302f998] 2024-11-10T22:38:12,171 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d51b53151407a9b9ad82a60ad302f998#B#compaction#133 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T22:38:12,172 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/B/d42420e675a24a4e83fc614cce54cbba is 50, key is test_row_0/B:col10/1731278291204/Put/seqid=0 2024-11-10T22:38:12,174 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241110b683cccd50dc4a048fd0b7ab035ef13a_d51b53151407a9b9ad82a60ad302f998 store=[table=TestAcidGuarantees family=A region=d51b53151407a9b9ad82a60ad302f998] 2024-11-10T22:38:12,179 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241110b683cccd50dc4a048fd0b7ab035ef13a_d51b53151407a9b9ad82a60ad302f998, store=[table=TestAcidGuarantees family=A region=d51b53151407a9b9ad82a60ad302f998] 2024-11-10T22:38:12,180 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110b683cccd50dc4a048fd0b7ab035ef13a_d51b53151407a9b9ad82a60ad302f998 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d51b53151407a9b9ad82a60ad302f998] 2024-11-10T22:38:12,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111046de3a749ac34eafb93d85e77c38f4f4_d51b53151407a9b9ad82a60ad302f998 is 50, key is test_row_0/A:col10/1731278291304/Put/seqid=0 2024-11-10T22:38:12,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073741975_1151 (size=12104) 2024-11-10T22:38:12,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073741976_1152 (size=4469) 2024-11-10T22:38:12,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073741977_1153 (size=12154) 2024-11-10T22:38:12,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:12,237 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111046de3a749ac34eafb93d85e77c38f4f4_d51b53151407a9b9ad82a60ad302f998 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111046de3a749ac34eafb93d85e77c38f4f4_d51b53151407a9b9ad82a60ad302f998 2024-11-10T22:38:12,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/A/099cb422e84c476db3a97644ce788de9, store: [table=TestAcidGuarantees family=A region=d51b53151407a9b9ad82a60ad302f998] 2024-11-10T22:38:12,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/A/099cb422e84c476db3a97644ce788de9 is 175, key is test_row_0/A:col10/1731278291304/Put/seqid=0 2024-11-10T22:38:12,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073741978_1154 (size=30955) 2024-11-10T22:38:12,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(8855): Flush requested on d51b53151407a9b9ad82a60ad302f998 2024-11-10T22:38:12,446 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. as already flushing 2024-11-10T22:38:12,469 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:12,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40778 deadline: 1731278352465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:12,469 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:12,469 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:12,470 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:12,472 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:12,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40774 deadline: 1731278352466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:12,473 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:12,473 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:12,473 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:12,473 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:12,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40738 deadline: 1731278352467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:12,473 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:12,473 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:12,473 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:12,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40754 deadline: 1731278352468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:12,473 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:12,473 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:12,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40736 deadline: 1731278352470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:12,474 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:12,474 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:12,474 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:12,474 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:12,474 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:12,474 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:12,577 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:12,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40778 deadline: 1731278352573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:12,578 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:12,578 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:12,578 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:12,584 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:12,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40754 deadline: 1731278352583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:12,585 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:12,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40774 deadline: 1731278352584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:12,585 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:12,585 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:12,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40738 deadline: 1731278352584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:12,586 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:12,586 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:12,586 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:12,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40736 deadline: 1731278352584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:12,585 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:12,587 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:12,587 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:12,587 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:12,586 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:12,586 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:12,588 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:12,588 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:12,588 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:12,615 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/B/d42420e675a24a4e83fc614cce54cbba as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B/d42420e675a24a4e83fc614cce54cbba 2024-11-10T22:38:12,623 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d51b53151407a9b9ad82a60ad302f998/B of d51b53151407a9b9ad82a60ad302f998 into d42420e675a24a4e83fc614cce54cbba(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T22:38:12,623 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d51b53151407a9b9ad82a60ad302f998: 2024-11-10T22:38:12,624 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., storeName=d51b53151407a9b9ad82a60ad302f998/B, priority=13, startTime=1731278292139; duration=0sec 2024-11-10T22:38:12,624 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d51b53151407a9b9ad82a60ad302f998#A#compaction#132 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T22:38:12,625 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T22:38:12,625 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d51b53151407a9b9ad82a60ad302f998:B 2024-11-10T22:38:12,625 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/A/a21a3de589ea499aadc8772b3865ef6c is 175, key is test_row_0/A:col10/1731278291204/Put/seqid=0 2024-11-10T22:38:12,625 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T22:38:12,627 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T22:38:12,627 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1541): d51b53151407a9b9ad82a60ad302f998/C is initiating minor compaction (all files) 2024-11-10T22:38:12,627 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d51b53151407a9b9ad82a60ad302f998/C in TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. 2024-11-10T22:38:12,627 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/C/fcfc690455ef47aca9ba62bf8c8e56a0, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/C/fa96840e58464ec8b93c24eeb453d3e8, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/C/65aeccd962204855b671817c73e75948] into tmpdir=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp, totalSize=35.2 K 2024-11-10T22:38:12,627 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting fcfc690455ef47aca9ba62bf8c8e56a0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731278289842 2024-11-10T22:38:12,628 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting fa96840e58464ec8b93c24eeb453d3e8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1731278289898 2024-11-10T22:38:12,628 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting 65aeccd962204855b671817c73e75948, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731278290577 2024-11-10T22:38:12,637 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073741979_1155 (size=31058) 2024-11-10T22:38:12,642 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d51b53151407a9b9ad82a60ad302f998#C#compaction#135 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T22:38:12,643 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/C/c35e8b7199014015a6c53944b35576ec is 50, key is test_row_0/C:col10/1731278291204/Put/seqid=0 2024-11-10T22:38:12,647 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=78, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/A/099cb422e84c476db3a97644ce788de9 2024-11-10T22:38:12,650 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/A/a21a3de589ea499aadc8772b3865ef6c as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/a21a3de589ea499aadc8772b3865ef6c 2024-11-10T22:38:12,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/B/13698c4362374b62912bd5df301b5fa9 is 50, key is test_row_0/B:col10/1731278291304/Put/seqid=0 2024-11-10T22:38:12,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073741980_1156 (size=12104) 2024-11-10T22:38:12,664 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d51b53151407a9b9ad82a60ad302f998/A of d51b53151407a9b9ad82a60ad302f998 into a21a3de589ea499aadc8772b3865ef6c(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T22:38:12,664 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d51b53151407a9b9ad82a60ad302f998: 2024-11-10T22:38:12,664 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., storeName=d51b53151407a9b9ad82a60ad302f998/A, priority=13, startTime=1731278292138; duration=0sec 2024-11-10T22:38:12,664 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T22:38:12,664 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d51b53151407a9b9ad82a60ad302f998:A 2024-11-10T22:38:12,676 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/C/c35e8b7199014015a6c53944b35576ec as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/C/c35e8b7199014015a6c53944b35576ec 2024-11-10T22:38:12,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073741981_1157 (size=12001) 2024-11-10T22:38:12,681 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/B/13698c4362374b62912bd5df301b5fa9 2024-11-10T22:38:12,686 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d51b53151407a9b9ad82a60ad302f998/C of d51b53151407a9b9ad82a60ad302f998 into c35e8b7199014015a6c53944b35576ec(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T22:38:12,686 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d51b53151407a9b9ad82a60ad302f998: 2024-11-10T22:38:12,686 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., storeName=d51b53151407a9b9ad82a60ad302f998/C, priority=13, startTime=1731278292139; duration=0sec 2024-11-10T22:38:12,686 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T22:38:12,686 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d51b53151407a9b9ad82a60ad302f998:C 2024-11-10T22:38:12,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/C/92e6e9dd106e48cabd630f7afa571ba4 is 50, key is test_row_0/C:col10/1731278291304/Put/seqid=0 2024-11-10T22:38:12,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073741982_1158 (size=12001) 2024-11-10T22:38:12,700 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/C/92e6e9dd106e48cabd630f7afa571ba4 2024-11-10T22:38:12,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/A/099cb422e84c476db3a97644ce788de9 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/099cb422e84c476db3a97644ce788de9 2024-11-10T22:38:12,713 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/099cb422e84c476db3a97644ce788de9, entries=150, sequenceid=78, filesize=30.2 K 2024-11-10T22:38:12,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/B/13698c4362374b62912bd5df301b5fa9 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B/13698c4362374b62912bd5df301b5fa9 2024-11-10T22:38:12,720 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B/13698c4362374b62912bd5df301b5fa9, entries=150, sequenceid=78, filesize=11.7 K 2024-11-10T22:38:12,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/C/92e6e9dd106e48cabd630f7afa571ba4 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/C/92e6e9dd106e48cabd630f7afa571ba4 2024-11-10T22:38:12,728 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/C/92e6e9dd106e48cabd630f7afa571ba4, entries=150, sequenceid=78, filesize=11.7 K 2024-11-10T22:38:12,730 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=67.09 KB/68700 for d51b53151407a9b9ad82a60ad302f998 in 569ms, sequenceid=78, compaction requested=false 2024-11-10T22:38:12,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for d51b53151407a9b9ad82a60ad302f998: 2024-11-10T22:38:12,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. 2024-11-10T22:38:12,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-11-10T22:38:12,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-11-10T22:38:12,737 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-11-10T22:38:12,737 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.8980 sec 2024-11-10T22:38:12,742 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 2.9100 sec 2024-11-10T22:38:12,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(8855): Flush requested on d51b53151407a9b9ad82a60ad302f998 2024-11-10T22:38:12,786 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T22:38:12,786 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d51b53151407a9b9ad82a60ad302f998 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-10T22:38:12,799 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411103d06650a1bcd4d429b21d2a092a72553_d51b53151407a9b9ad82a60ad302f998 is 50, key is test_row_0/A:col10/1731278292785/Put/seqid=0 2024-11-10T22:38:12,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073741983_1159 (size=14594) 2024-11-10T22:38:12,807 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:12,813 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411103d06650a1bcd4d429b21d2a092a72553_d51b53151407a9b9ad82a60ad302f998 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411103d06650a1bcd4d429b21d2a092a72553_d51b53151407a9b9ad82a60ad302f998 2024-11-10T22:38:12,815 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/A/16860bcb95a34b6487878236e7302a1c, store: [table=TestAcidGuarantees family=A region=d51b53151407a9b9ad82a60ad302f998] 2024-11-10T22:38:12,816 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/A/16860bcb95a34b6487878236e7302a1c is 175, key is test_row_0/A:col10/1731278292785/Put/seqid=0 2024-11-10T22:38:12,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073741984_1160 (size=39549) 2024-11-10T22:38:12,827 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:12,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40736 deadline: 1731278352818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:12,828 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:12,828 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:12,828 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:12,828 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:12,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40778 deadline: 1731278352819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:12,829 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=95, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/A/16860bcb95a34b6487878236e7302a1c 2024-11-10T22:38:12,829 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:12,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40754 deadline: 1731278352821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:12,830 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:12,830 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:12,830 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:12,830 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:12,830 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:12,830 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:12,832 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:12,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40774 deadline: 1731278352828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:12,833 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:12,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40738 deadline: 1731278352829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:12,833 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:12,833 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:12,833 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:12,834 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:12,834 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:12,834 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:12,843 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/B/97e2bd2d217443348f4734caba5f6c63 is 50, key is test_row_0/B:col10/1731278292785/Put/seqid=0 2024-11-10T22:38:12,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073741985_1161 (size=12001) 2024-11-10T22:38:12,866 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/B/97e2bd2d217443348f4734caba5f6c63 2024-11-10T22:38:12,878 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/C/db7c10918d2846f2bd06dd5aab55bd32 is 50, key is test_row_0/C:col10/1731278292785/Put/seqid=0 2024-11-10T22:38:12,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073741986_1162 (size=12001) 2024-11-10T22:38:12,890 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/C/db7c10918d2846f2bd06dd5aab55bd32 2024-11-10T22:38:12,899 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/A/16860bcb95a34b6487878236e7302a1c as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/16860bcb95a34b6487878236e7302a1c 2024-11-10T22:38:12,904 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/16860bcb95a34b6487878236e7302a1c, entries=200, sequenceid=95, filesize=38.6 K 2024-11-10T22:38:12,905 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/B/97e2bd2d217443348f4734caba5f6c63 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B/97e2bd2d217443348f4734caba5f6c63 2024-11-10T22:38:12,910 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B/97e2bd2d217443348f4734caba5f6c63, entries=150, sequenceid=95, filesize=11.7 K 2024-11-10T22:38:12,911 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/C/db7c10918d2846f2bd06dd5aab55bd32 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/C/db7c10918d2846f2bd06dd5aab55bd32 2024-11-10T22:38:12,920 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/C/db7c10918d2846f2bd06dd5aab55bd32, entries=150, sequenceid=95, filesize=11.7 K 2024-11-10T22:38:12,924 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=134.18 KB/137400 for d51b53151407a9b9ad82a60ad302f998 in 137ms, sequenceid=95, compaction requested=true 2024-11-10T22:38:12,924 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d51b53151407a9b9ad82a60ad302f998: 2024-11-10T22:38:12,924 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d51b53151407a9b9ad82a60ad302f998:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T22:38:12,924 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T22:38:12,924 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T22:38:12,924 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T22:38:12,924 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d51b53151407a9b9ad82a60ad302f998:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T22:38:12,924 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T22:38:12,924 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d51b53151407a9b9ad82a60ad302f998:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T22:38:12,924 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T22:38:12,925 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T22:38:12,925 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1541): d51b53151407a9b9ad82a60ad302f998/B is initiating minor compaction (all files) 2024-11-10T22:38:12,926 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d51b53151407a9b9ad82a60ad302f998/B in TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. 2024-11-10T22:38:12,926 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B/d42420e675a24a4e83fc614cce54cbba, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B/13698c4362374b62912bd5df301b5fa9, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B/97e2bd2d217443348f4734caba5f6c63] into tmpdir=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp, totalSize=35.3 K 2024-11-10T22:38:12,926 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101562 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T22:38:12,926 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HStore(1541): d51b53151407a9b9ad82a60ad302f998/A is initiating minor compaction (all files) 2024-11-10T22:38:12,926 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d51b53151407a9b9ad82a60ad302f998/A in TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. 2024-11-10T22:38:12,926 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/a21a3de589ea499aadc8772b3865ef6c, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/099cb422e84c476db3a97644ce788de9, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/16860bcb95a34b6487878236e7302a1c] into tmpdir=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp, totalSize=99.2 K 2024-11-10T22:38:12,926 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. 2024-11-10T22:38:12,926 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. files: [hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/a21a3de589ea499aadc8772b3865ef6c, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/099cb422e84c476db3a97644ce788de9, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/16860bcb95a34b6487878236e7302a1c] 2024-11-10T22:38:12,927 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting d42420e675a24a4e83fc614cce54cbba, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731278290577 2024-11-10T22:38:12,927 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting a21a3de589ea499aadc8772b3865ef6c, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731278290577 2024-11-10T22:38:12,927 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting 13698c4362374b62912bd5df301b5fa9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1731278291302 2024-11-10T22:38:12,928 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 099cb422e84c476db3a97644ce788de9, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1731278291302 2024-11-10T22:38:12,928 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting 97e2bd2d217443348f4734caba5f6c63, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731278292445 2024-11-10T22:38:12,928 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 16860bcb95a34b6487878236e7302a1c, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731278292445 2024-11-10T22:38:12,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(8855): Flush requested on d51b53151407a9b9ad82a60ad302f998 2024-11-10T22:38:12,935 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T22:38:12,935 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d51b53151407a9b9ad82a60ad302f998 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-10T22:38:12,943 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d51b53151407a9b9ad82a60ad302f998] 2024-11-10T22:38:12,954 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d51b53151407a9b9ad82a60ad302f998#B#compaction#142 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T22:38:12,954 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/B/ad445e689641457dbfbd04005208429c is 50, key is test_row_0/B:col10/1731278292785/Put/seqid=0 2024-11-10T22:38:12,955 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411108fcd8e4170704e81bab5c4b04544c968_d51b53151407a9b9ad82a60ad302f998 store=[table=TestAcidGuarantees family=A region=d51b53151407a9b9ad82a60ad302f998] 2024-11-10T22:38:12,954 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:12,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40774 deadline: 1731278352949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:12,955 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:12,955 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:12,955 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:12,955 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:12,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40754 deadline: 1731278352950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:12,956 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:12,956 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:12,956 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:12,956 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:12,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40738 deadline: 1731278352951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:12,956 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:12,956 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:12,957 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:12,957 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411108fcd8e4170704e81bab5c4b04544c968_d51b53151407a9b9ad82a60ad302f998, store=[table=TestAcidGuarantees family=A region=d51b53151407a9b9ad82a60ad302f998] 2024-11-10T22:38:12,957 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411108fcd8e4170704e81bab5c4b04544c968_d51b53151407a9b9ad82a60ad302f998 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d51b53151407a9b9ad82a60ad302f998] 2024-11-10T22:38:12,959 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:12,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40736 deadline: 1731278352958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:12,959 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:12,959 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:12,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40778 deadline: 1731278352959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:12,959 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:12,959 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:12,960 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:12,960 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:12,960 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:12,966 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110fbc328f01ec64b28b5929ac08b6ecae8_d51b53151407a9b9ad82a60ad302f998 is 50, key is test_row_0/A:col10/1731278292935/Put/seqid=0 2024-11-10T22:38:12,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073741987_1163 (size=12207) 2024-11-10T22:38:12,990 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/B/ad445e689641457dbfbd04005208429c as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B/ad445e689641457dbfbd04005208429c 2024-11-10T22:38:12,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073741988_1164 (size=17034) 2024-11-10T22:38:12,997 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d51b53151407a9b9ad82a60ad302f998/B of d51b53151407a9b9ad82a60ad302f998 into ad445e689641457dbfbd04005208429c(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T22:38:12,997 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d51b53151407a9b9ad82a60ad302f998: 2024-11-10T22:38:12,997 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., storeName=d51b53151407a9b9ad82a60ad302f998/B, priority=13, startTime=1731278292924; duration=0sec 2024-11-10T22:38:12,997 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T22:38:12,997 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d51b53151407a9b9ad82a60ad302f998:B 2024-11-10T22:38:12,997 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T22:38:12,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073741989_1165 (size=4469) 2024-11-10T22:38:12,999 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T22:38:12,999 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1541): d51b53151407a9b9ad82a60ad302f998/C is initiating minor compaction (all files) 2024-11-10T22:38:12,999 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d51b53151407a9b9ad82a60ad302f998/C in TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. 2024-11-10T22:38:12,999 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/C/c35e8b7199014015a6c53944b35576ec, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/C/92e6e9dd106e48cabd630f7afa571ba4, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/C/db7c10918d2846f2bd06dd5aab55bd32] into tmpdir=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp, totalSize=35.3 K 2024-11-10T22:38:12,999 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d51b53151407a9b9ad82a60ad302f998#A#compaction#141 average throughput is 0.44 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T22:38:13,000 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting c35e8b7199014015a6c53944b35576ec, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731278290577 2024-11-10T22:38:13,000 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/A/9036257355fb403bb6ed4fc7ce5b9efd is 175, key is test_row_0/A:col10/1731278292785/Put/seqid=0 2024-11-10T22:38:13,001 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting 92e6e9dd106e48cabd630f7afa571ba4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1731278291302 2024-11-10T22:38:13,002 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting db7c10918d2846f2bd06dd5aab55bd32, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731278292445 2024-11-10T22:38:13,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073741990_1166 (size=31161) 2024-11-10T22:38:13,009 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d51b53151407a9b9ad82a60ad302f998#C#compaction#144 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T22:38:13,010 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/C/252832697c754b54856d36238cdac8f1 is 50, key is test_row_0/C:col10/1731278292785/Put/seqid=0 2024-11-10T22:38:13,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073741991_1167 (size=12207) 2024-11-10T22:38:13,062 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:13,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40774 deadline: 1731278353062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:13,063 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:13,063 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:13,063 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:13,063 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:13,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40754 deadline: 1731278353062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:13,064 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:13,064 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:13,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40738 deadline: 1731278353062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:13,064 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:13,064 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:13,064 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:13,064 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:13,064 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:13,064 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:13,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40778 deadline: 1731278353062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:13,064 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:13,064 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:13,064 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:13,064 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:13,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40736 deadline: 1731278353063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:13,065 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:13,065 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:13,065 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:13,275 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:13,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40774 deadline: 1731278353272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:13,276 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:13,276 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:13,276 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:13,276 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:13,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40778 deadline: 1731278353273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:13,276 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:13,276 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:13,276 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:13,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40754 deadline: 1731278353273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:13,276 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:13,277 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:13,277 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:13,277 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:13,277 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:13,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40738 deadline: 1731278353273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:13,277 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:13,277 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:13,277 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:13,277 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:13,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40736 deadline: 1731278353273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:13,278 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:13,278 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:13,278 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:13,396 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:13,402 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110fbc328f01ec64b28b5929ac08b6ecae8_d51b53151407a9b9ad82a60ad302f998 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110fbc328f01ec64b28b5929ac08b6ecae8_d51b53151407a9b9ad82a60ad302f998 2024-11-10T22:38:13,403 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/A/896a89738257408ab02f7a4df7b0bac9, store: [table=TestAcidGuarantees family=A region=d51b53151407a9b9ad82a60ad302f998] 2024-11-10T22:38:13,404 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/A/896a89738257408ab02f7a4df7b0bac9 is 175, key is test_row_0/A:col10/1731278292935/Put/seqid=0 2024-11-10T22:38:13,412 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/A/9036257355fb403bb6ed4fc7ce5b9efd as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/9036257355fb403bb6ed4fc7ce5b9efd 2024-11-10T22:38:13,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073741992_1168 (size=48139) 2024-11-10T22:38:13,418 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=120, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/A/896a89738257408ab02f7a4df7b0bac9 2024-11-10T22:38:13,420 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d51b53151407a9b9ad82a60ad302f998/A of d51b53151407a9b9ad82a60ad302f998 into 9036257355fb403bb6ed4fc7ce5b9efd(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T22:38:13,420 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d51b53151407a9b9ad82a60ad302f998: 2024-11-10T22:38:13,420 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., storeName=d51b53151407a9b9ad82a60ad302f998/A, priority=13, startTime=1731278292924; duration=0sec 2024-11-10T22:38:13,420 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T22:38:13,420 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d51b53151407a9b9ad82a60ad302f998:A 2024-11-10T22:38:13,428 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/C/252832697c754b54856d36238cdac8f1 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/C/252832697c754b54856d36238cdac8f1 2024-11-10T22:38:13,431 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/B/33835994db69410ea57ea2a9ce009173 is 50, key is test_row_0/B:col10/1731278292935/Put/seqid=0 2024-11-10T22:38:13,435 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d51b53151407a9b9ad82a60ad302f998/C of d51b53151407a9b9ad82a60ad302f998 into 252832697c754b54856d36238cdac8f1(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T22:38:13,435 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d51b53151407a9b9ad82a60ad302f998: 2024-11-10T22:38:13,435 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., storeName=d51b53151407a9b9ad82a60ad302f998/C, priority=13, startTime=1731278292924; duration=0sec 2024-11-10T22:38:13,435 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T22:38:13,435 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d51b53151407a9b9ad82a60ad302f998:C 2024-11-10T22:38:13,444 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073741993_1169 (size=12001) 2024-11-10T22:38:13,444 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=120 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/B/33835994db69410ea57ea2a9ce009173 2024-11-10T22:38:13,454 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/C/29e0ac2d74a04bfa829bd22869033515 is 50, key is test_row_0/C:col10/1731278292935/Put/seqid=0 2024-11-10T22:38:13,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073741994_1170 (size=12001) 2024-11-10T22:38:13,467 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=120 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/C/29e0ac2d74a04bfa829bd22869033515 2024-11-10T22:38:13,473 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/A/896a89738257408ab02f7a4df7b0bac9 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/896a89738257408ab02f7a4df7b0bac9 2024-11-10T22:38:13,478 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/896a89738257408ab02f7a4df7b0bac9, entries=250, sequenceid=120, filesize=47.0 K 2024-11-10T22:38:13,479 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/B/33835994db69410ea57ea2a9ce009173 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B/33835994db69410ea57ea2a9ce009173 2024-11-10T22:38:13,485 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B/33835994db69410ea57ea2a9ce009173, entries=150, sequenceid=120, filesize=11.7 K 2024-11-10T22:38:13,487 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/C/29e0ac2d74a04bfa829bd22869033515 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/C/29e0ac2d74a04bfa829bd22869033515 2024-11-10T22:38:13,493 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/C/29e0ac2d74a04bfa829bd22869033515, entries=150, sequenceid=120, filesize=11.7 K 2024-11-10T22:38:13,495 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for d51b53151407a9b9ad82a60ad302f998 in 559ms, sequenceid=120, compaction requested=false 2024-11-10T22:38:13,495 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d51b53151407a9b9ad82a60ad302f998: 2024-11-10T22:38:13,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(8855): Flush requested on d51b53151407a9b9ad82a60ad302f998 2024-11-10T22:38:13,587 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T22:38:13,587 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d51b53151407a9b9ad82a60ad302f998 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-10T22:38:13,603 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111084757fae3322484d891f600a40019542_d51b53151407a9b9ad82a60ad302f998 is 50, key is test_row_0/A:col10/1731278292953/Put/seqid=0 2024-11-10T22:38:13,621 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:13,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40778 deadline: 1731278353615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:13,622 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:13,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40736 deadline: 1731278353618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:13,622 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:13,622 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:13,623 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:13,624 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:13,624 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:13,624 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:13,626 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:13,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40754 deadline: 1731278353620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:13,626 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:13,626 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:13,626 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:13,628 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:13,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40738 deadline: 1731278353622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:13,629 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:13,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40774 deadline: 1731278353622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:13,629 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:13,629 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:13,629 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:13,629 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:13,629 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:13,629 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:13,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073741995_1171 (size=12254) 2024-11-10T22:38:13,633 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:13,638 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111084757fae3322484d891f600a40019542_d51b53151407a9b9ad82a60ad302f998 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111084757fae3322484d891f600a40019542_d51b53151407a9b9ad82a60ad302f998 2024-11-10T22:38:13,639 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/A/a5d1a25527f24defaea12e3250b0e326, store: [table=TestAcidGuarantees family=A region=d51b53151407a9b9ad82a60ad302f998] 2024-11-10T22:38:13,640 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/A/a5d1a25527f24defaea12e3250b0e326 is 175, key is test_row_0/A:col10/1731278292953/Put/seqid=0 2024-11-10T22:38:13,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073741996_1172 (size=31055) 2024-11-10T22:38:13,665 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=136, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/A/a5d1a25527f24defaea12e3250b0e326 2024-11-10T22:38:13,686 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/B/db2538418a264226914ded5758134157 is 50, key is test_row_0/B:col10/1731278292953/Put/seqid=0 2024-11-10T22:38:13,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073741997_1173 (size=12101) 2024-11-10T22:38:13,694 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=136 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/B/db2538418a264226914ded5758134157 2024-11-10T22:38:13,713 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/C/d27e9e1796f7479cb3919f9aec3b8726 is 50, key is test_row_0/C:col10/1731278292953/Put/seqid=0 2024-11-10T22:38:13,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073741998_1174 (size=12101) 2024-11-10T22:38:13,728 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=136 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/C/d27e9e1796f7479cb3919f9aec3b8726 2024-11-10T22:38:13,734 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:13,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40736 deadline: 1731278353734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:13,735 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:13,735 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:13,735 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:13,735 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/A/a5d1a25527f24defaea12e3250b0e326 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/a5d1a25527f24defaea12e3250b0e326 2024-11-10T22:38:13,736 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:13,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40738 deadline: 1731278353734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:13,736 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:13,736 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:13,736 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:13,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40774 deadline: 1731278353734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:13,736 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:13,737 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:13,737 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:13,737 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:13,740 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:13,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40778 deadline: 1731278353734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:13,741 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:13,741 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:13,741 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:13,741 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:13,742 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/a5d1a25527f24defaea12e3250b0e326, entries=150, sequenceid=136, filesize=30.3 K 2024-11-10T22:38:13,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40754 deadline: 1731278353734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:13,742 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:13,742 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:13,742 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:13,743 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/B/db2538418a264226914ded5758134157 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B/db2538418a264226914ded5758134157 2024-11-10T22:38:13,747 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B/db2538418a264226914ded5758134157, entries=150, sequenceid=136, filesize=11.8 K 2024-11-10T22:38:13,748 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/C/d27e9e1796f7479cb3919f9aec3b8726 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/C/d27e9e1796f7479cb3919f9aec3b8726 2024-11-10T22:38:13,754 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/C/d27e9e1796f7479cb3919f9aec3b8726, entries=150, sequenceid=136, filesize=11.8 K 2024-11-10T22:38:13,755 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for d51b53151407a9b9ad82a60ad302f998 in 168ms, sequenceid=136, compaction requested=true 2024-11-10T22:38:13,755 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d51b53151407a9b9ad82a60ad302f998: 2024-11-10T22:38:13,756 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T22:38:13,756 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d51b53151407a9b9ad82a60ad302f998:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T22:38:13,756 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T22:38:13,756 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T22:38:13,756 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d51b53151407a9b9ad82a60ad302f998:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T22:38:13,756 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T22:38:13,756 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d51b53151407a9b9ad82a60ad302f998:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T22:38:13,756 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T22:38:13,758 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110355 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T22:38:13,758 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HStore(1541): d51b53151407a9b9ad82a60ad302f998/A is initiating minor compaction (all files) 2024-11-10T22:38:13,758 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d51b53151407a9b9ad82a60ad302f998/A in TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. 2024-11-10T22:38:13,758 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/9036257355fb403bb6ed4fc7ce5b9efd, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/896a89738257408ab02f7a4df7b0bac9, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/a5d1a25527f24defaea12e3250b0e326] into tmpdir=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp, totalSize=107.8 K 2024-11-10T22:38:13,758 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. 2024-11-10T22:38:13,758 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. files: [hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/9036257355fb403bb6ed4fc7ce5b9efd, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/896a89738257408ab02f7a4df7b0bac9, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/a5d1a25527f24defaea12e3250b0e326] 2024-11-10T22:38:13,759 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9036257355fb403bb6ed4fc7ce5b9efd, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731278292445 2024-11-10T22:38:13,761 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 896a89738257408ab02f7a4df7b0bac9, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1731278292815 2024-11-10T22:38:13,762 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T22:38:13,762 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1541): d51b53151407a9b9ad82a60ad302f998/B is initiating minor compaction (all files) 2024-11-10T22:38:13,762 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d51b53151407a9b9ad82a60ad302f998/B in TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. 2024-11-10T22:38:13,762 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B/ad445e689641457dbfbd04005208429c, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B/33835994db69410ea57ea2a9ce009173, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B/db2538418a264226914ded5758134157] into tmpdir=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp, totalSize=35.5 K 2024-11-10T22:38:13,762 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting a5d1a25527f24defaea12e3250b0e326, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1731278292948 2024-11-10T22:38:13,763 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting ad445e689641457dbfbd04005208429c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731278292445 2024-11-10T22:38:13,763 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting 33835994db69410ea57ea2a9ce009173, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1731278292825 2024-11-10T22:38:13,764 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting db2538418a264226914ded5758134157, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1731278292948 2024-11-10T22:38:13,778 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d51b53151407a9b9ad82a60ad302f998#B#compaction#150 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T22:38:13,779 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/B/79f0d3719e57470d8fb01c738594f277 is 50, key is test_row_0/B:col10/1731278292953/Put/seqid=0 2024-11-10T22:38:13,780 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d51b53151407a9b9ad82a60ad302f998] 2024-11-10T22:38:13,788 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241110f24217b73e834a838b8aeedd50457e70_d51b53151407a9b9ad82a60ad302f998 store=[table=TestAcidGuarantees family=A region=d51b53151407a9b9ad82a60ad302f998] 2024-11-10T22:38:13,791 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241110f24217b73e834a838b8aeedd50457e70_d51b53151407a9b9ad82a60ad302f998, store=[table=TestAcidGuarantees family=A region=d51b53151407a9b9ad82a60ad302f998] 2024-11-10T22:38:13,791 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110f24217b73e834a838b8aeedd50457e70_d51b53151407a9b9ad82a60ad302f998 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d51b53151407a9b9ad82a60ad302f998] 2024-11-10T22:38:13,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073741999_1175 (size=12409) 2024-11-10T22:38:13,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742000_1176 (size=4469) 2024-11-10T22:38:13,812 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/B/79f0d3719e57470d8fb01c738594f277 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B/79f0d3719e57470d8fb01c738594f277 2024-11-10T22:38:13,818 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d51b53151407a9b9ad82a60ad302f998/B of d51b53151407a9b9ad82a60ad302f998 into 79f0d3719e57470d8fb01c738594f277(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T22:38:13,818 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d51b53151407a9b9ad82a60ad302f998: 2024-11-10T22:38:13,818 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., storeName=d51b53151407a9b9ad82a60ad302f998/B, priority=13, startTime=1731278293756; duration=0sec 2024-11-10T22:38:13,819 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T22:38:13,819 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d51b53151407a9b9ad82a60ad302f998:B 2024-11-10T22:38:13,819 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T22:38:13,820 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T22:38:13,820 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1541): d51b53151407a9b9ad82a60ad302f998/C is initiating minor compaction (all files) 2024-11-10T22:38:13,820 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d51b53151407a9b9ad82a60ad302f998/C in TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. 2024-11-10T22:38:13,820 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/C/252832697c754b54856d36238cdac8f1, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/C/29e0ac2d74a04bfa829bd22869033515, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/C/d27e9e1796f7479cb3919f9aec3b8726] into tmpdir=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp, totalSize=35.5 K 2024-11-10T22:38:13,821 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting 252832697c754b54856d36238cdac8f1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731278292445 2024-11-10T22:38:13,822 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting 29e0ac2d74a04bfa829bd22869033515, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1731278292825 2024-11-10T22:38:13,822 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting d27e9e1796f7479cb3919f9aec3b8726, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1731278292948 2024-11-10T22:38:13,831 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d51b53151407a9b9ad82a60ad302f998#C#compaction#152 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T22:38:13,834 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/C/35bee95a01c640aeb8ac8cd0c20c0b9e is 50, key is test_row_0/C:col10/1731278292953/Put/seqid=0 2024-11-10T22:38:13,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742001_1177 (size=12409) 2024-11-10T22:38:13,867 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/C/35bee95a01c640aeb8ac8cd0c20c0b9e as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/C/35bee95a01c640aeb8ac8cd0c20c0b9e 2024-11-10T22:38:13,877 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d51b53151407a9b9ad82a60ad302f998/C of d51b53151407a9b9ad82a60ad302f998 into 35bee95a01c640aeb8ac8cd0c20c0b9e(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T22:38:13,877 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d51b53151407a9b9ad82a60ad302f998: 2024-11-10T22:38:13,877 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., storeName=d51b53151407a9b9ad82a60ad302f998/C, priority=13, startTime=1731278293756; duration=0sec 2024-11-10T22:38:13,877 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T22:38:13,878 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d51b53151407a9b9ad82a60ad302f998:C 2024-11-10T22:38:13,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(8855): Flush requested on d51b53151407a9b9ad82a60ad302f998 2024-11-10T22:38:13,974 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T22:38:13,974 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d51b53151407a9b9ad82a60ad302f998 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-10T22:38:13,982 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110fdc393325c704a8996b5a6e72bd89621_d51b53151407a9b9ad82a60ad302f998 is 50, key is test_row_0/A:col10/1731278293947/Put/seqid=0 2024-11-10T22:38:13,987 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:13,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40736 deadline: 1731278353985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:13,988 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:13,988 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:13,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40778 deadline: 1731278353986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:13,988 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:13,988 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:13,988 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:13,988 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:13,988 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:13,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742002_1178 (size=12304) 2024-11-10T22:38:13,994 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:13,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40738 deadline: 1731278353987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:13,995 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:13,995 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:13,995 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:13,995 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:13,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40754 deadline: 1731278353988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:13,995 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:13,995 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:13,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40774 deadline: 1731278353988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:13,995 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:13,995 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:13,995 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:13,995 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:13,995 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:14,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-10T22:38:14,003 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T22:38:14,003 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T22:38:14,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-11-10T22:38:14,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-10T22:38:14,005 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T22:38:14,006 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T22:38:14,006 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T22:38:14,093 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:14,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40736 deadline: 1731278354092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:14,093 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:14,094 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:14,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40778 deadline: 1731278354093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:14,094 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:14,094 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:14,094 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:14,094 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:14,094 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:14,103 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:14,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40738 deadline: 1731278354102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:14,104 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:14,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40754 deadline: 1731278354102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:14,104 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:14,104 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:14,105 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:14,104 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:14,105 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:14,105 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:14,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40774 deadline: 1731278354102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:14,105 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:14,105 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:14,105 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:14,105 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:14,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-10T22:38:14,160 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-10T22:38:14,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. 2024-11-10T22:38:14,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. as already flushing 2024-11-10T22:38:14,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. 2024-11-10T22:38:14,160 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => d51b53151407a9b9ad82a60ad302f998, NAME => 'TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:14,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => d51b53151407a9b9ad82a60ad302f998, NAME => 'TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:14,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d51b53151407a9b9ad82a60ad302f998, NAME => 'TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d51b53151407a9b9ad82a60ad302f998, NAME => 'TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:14,211 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d51b53151407a9b9ad82a60ad302f998#A#compaction#151 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T22:38:14,212 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/A/8810867aca3b42af8c217923a96327aa is 175, key is test_row_0/A:col10/1731278292953/Put/seqid=0 2024-11-10T22:38:14,216 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742003_1179 (size=31363) 2024-11-10T22:38:14,306 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:14,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40736 deadline: 1731278354302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:14,306 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:14,306 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:14,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40778 deadline: 1731278354302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:14,306 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:14,306 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:14,307 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:14,307 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:14,307 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:14,315 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:14,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40738 deadline: 1731278354315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:14,315 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-10T22:38:14,315 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:14,315 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:14,316 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:14,316 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:14,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. 2024-11-10T22:38:14,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40774 deadline: 1731278354315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:14,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. as already flushing 2024-11-10T22:38:14,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. 2024-11-10T22:38:14,316 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => d51b53151407a9b9ad82a60ad302f998, NAME => 'TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:14,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => d51b53151407a9b9ad82a60ad302f998, NAME => 'TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:14,317 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:14,317 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:14,317 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:14,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d51b53151407a9b9ad82a60ad302f998, NAME => 'TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d51b53151407a9b9ad82a60ad302f998, NAME => 'TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:14,319 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:14,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40754 deadline: 1731278354316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:14,320 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:14,320 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:14,320 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:14,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-10T22:38:14,390 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:14,396 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110fdc393325c704a8996b5a6e72bd89621_d51b53151407a9b9ad82a60ad302f998 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110fdc393325c704a8996b5a6e72bd89621_d51b53151407a9b9ad82a60ad302f998 2024-11-10T22:38:14,398 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/A/359698e468ba47788d1b67fe67724389, store: [table=TestAcidGuarantees family=A region=d51b53151407a9b9ad82a60ad302f998] 2024-11-10T22:38:14,398 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/A/359698e468ba47788d1b67fe67724389 is 175, key is test_row_0/A:col10/1731278293947/Put/seqid=0 2024-11-10T22:38:14,413 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742004_1180 (size=31105) 2024-11-10T22:38:14,414 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=164, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/A/359698e468ba47788d1b67fe67724389 2024-11-10T22:38:14,432 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/B/c47796a3673f4344bbee360f081a9433 is 50, key is test_row_0/B:col10/1731278293947/Put/seqid=0 2024-11-10T22:38:14,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742005_1181 (size=12151) 2024-11-10T22:38:14,444 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=164 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/B/c47796a3673f4344bbee360f081a9433 2024-11-10T22:38:14,455 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/C/95e6a53bc8fc48d198df5c48e16ede98 is 50, key is test_row_0/C:col10/1731278293947/Put/seqid=0 2024-11-10T22:38:14,470 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-10T22:38:14,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. 2024-11-10T22:38:14,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. as already flushing 2024-11-10T22:38:14,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. 2024-11-10T22:38:14,471 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => d51b53151407a9b9ad82a60ad302f998, NAME => 'TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:14,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => d51b53151407a9b9ad82a60ad302f998, NAME => 'TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:14,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d51b53151407a9b9ad82a60ad302f998, NAME => 'TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d51b53151407a9b9ad82a60ad302f998, NAME => 'TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:14,475 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742006_1182 (size=12151) 2024-11-10T22:38:14,478 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=164 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/C/95e6a53bc8fc48d198df5c48e16ede98 2024-11-10T22:38:14,486 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/A/359698e468ba47788d1b67fe67724389 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/359698e468ba47788d1b67fe67724389 2024-11-10T22:38:14,496 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/359698e468ba47788d1b67fe67724389, entries=150, sequenceid=164, filesize=30.4 K 2024-11-10T22:38:14,498 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/B/c47796a3673f4344bbee360f081a9433 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B/c47796a3673f4344bbee360f081a9433 2024-11-10T22:38:14,508 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B/c47796a3673f4344bbee360f081a9433, entries=150, sequenceid=164, filesize=11.9 K 2024-11-10T22:38:14,511 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/C/95e6a53bc8fc48d198df5c48e16ede98 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/C/95e6a53bc8fc48d198df5c48e16ede98 2024-11-10T22:38:14,522 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/C/95e6a53bc8fc48d198df5c48e16ede98, entries=150, sequenceid=164, filesize=11.9 K 2024-11-10T22:38:14,523 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=60.38 KB/61830 for d51b53151407a9b9ad82a60ad302f998 in 549ms, sequenceid=164, compaction requested=false 2024-11-10T22:38:14,523 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d51b53151407a9b9ad82a60ad302f998: 2024-11-10T22:38:14,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(8855): Flush requested on d51b53151407a9b9ad82a60ad302f998 2024-11-10T22:38:14,615 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T22:38:14,616 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d51b53151407a9b9ad82a60ad302f998 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-10T22:38:14,623 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-10T22:38:14,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. 2024-11-10T22:38:14,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. as already flushing 2024-11-10T22:38:14,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. 2024-11-10T22:38:14,624 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => d51b53151407a9b9ad82a60ad302f998, NAME => 'TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:14,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => d51b53151407a9b9ad82a60ad302f998, NAME => 'TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:14,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d51b53151407a9b9ad82a60ad302f998, NAME => 'TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d51b53151407a9b9ad82a60ad302f998, NAME => 'TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:14,626 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/A/8810867aca3b42af8c217923a96327aa as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/8810867aca3b42af8c217923a96327aa 2024-11-10T22:38:14,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-10T22:38:14,636 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d51b53151407a9b9ad82a60ad302f998/A of d51b53151407a9b9ad82a60ad302f998 into 8810867aca3b42af8c217923a96327aa(size=30.6 K), total size for store is 61.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T22:38:14,636 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d51b53151407a9b9ad82a60ad302f998: 2024-11-10T22:38:14,637 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., storeName=d51b53151407a9b9ad82a60ad302f998/A, priority=13, startTime=1731278293755; duration=0sec 2024-11-10T22:38:14,637 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T22:38:14,637 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d51b53151407a9b9ad82a60ad302f998:A 2024-11-10T22:38:14,646 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111004d645ea63114b7b80e11a52e482e18d_d51b53151407a9b9ad82a60ad302f998 is 50, key is test_row_0/A:col10/1731278294614/Put/seqid=0 2024-11-10T22:38:14,651 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:14,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40774 deadline: 1731278354644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:14,652 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:14,652 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:14,652 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:14,655 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:14,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40778 deadline: 1731278354648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:14,656 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:14,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40736 deadline: 1731278354649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:14,656 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:14,656 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:14,656 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:14,656 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:14,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40754 deadline: 1731278354650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:14,657 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:14,657 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:14,657 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:14,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40738 deadline: 1731278354652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:14,657 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:14,657 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:14,657 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:14,657 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:14,657 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:14,657 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:14,658 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:14,671 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742007_1183 (size=17284) 2024-11-10T22:38:14,672 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:14,679 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111004d645ea63114b7b80e11a52e482e18d_d51b53151407a9b9ad82a60ad302f998 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111004d645ea63114b7b80e11a52e482e18d_d51b53151407a9b9ad82a60ad302f998 2024-11-10T22:38:14,681 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/A/3c53ea79ea674e708881fdc7007f15ac, store: [table=TestAcidGuarantees family=A region=d51b53151407a9b9ad82a60ad302f998] 2024-11-10T22:38:14,682 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/A/3c53ea79ea674e708881fdc7007f15ac is 175, key is test_row_0/A:col10/1731278294614/Put/seqid=0 2024-11-10T22:38:14,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742008_1184 (size=48389) 2024-11-10T22:38:14,694 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=178, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/A/3c53ea79ea674e708881fdc7007f15ac 2024-11-10T22:38:14,707 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/B/486fc2c33da9494f8fc3be85f64d8c5c is 50, key is test_row_0/B:col10/1731278294614/Put/seqid=0 2024-11-10T22:38:14,715 INFO [master/849d0d71310f:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-10T22:38:14,716 INFO [master/849d0d71310f:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-10T22:38:14,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742009_1185 (size=12151) 2024-11-10T22:38:14,723 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=178 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/B/486fc2c33da9494f8fc3be85f64d8c5c 2024-11-10T22:38:14,735 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/C/a8807d8fabb1426b86517d7fbd5027cb is 50, key is test_row_0/C:col10/1731278294614/Put/seqid=0 2024-11-10T22:38:14,747 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742010_1186 (size=12151) 2024-11-10T22:38:14,767 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:14,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40774 deadline: 1731278354762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:14,768 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:14,768 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:14,768 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:14,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40778 deadline: 1731278354763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:14,768 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:14,768 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:14,768 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:14,768 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:14,768 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:14,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40754 deadline: 1731278354763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:14,769 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:14,769 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:14,769 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:14,769 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:14,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40738 deadline: 1731278354763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:14,769 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:14,769 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:14,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40736 deadline: 1731278354763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:14,769 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:14,769 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:14,769 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:14,769 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:14,769 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:14,777 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-10T22:38:14,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. 2024-11-10T22:38:14,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. as already flushing 2024-11-10T22:38:14,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. 2024-11-10T22:38:14,777 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => d51b53151407a9b9ad82a60ad302f998, NAME => 'TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:14,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => d51b53151407a9b9ad82a60ad302f998, NAME => 'TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:14,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d51b53151407a9b9ad82a60ad302f998, NAME => 'TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d51b53151407a9b9ad82a60ad302f998, NAME => 'TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:14,929 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-10T22:38:14,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. 2024-11-10T22:38:14,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. as already flushing 2024-11-10T22:38:14,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. 2024-11-10T22:38:14,930 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => d51b53151407a9b9ad82a60ad302f998, NAME => 'TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:14,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => d51b53151407a9b9ad82a60ad302f998, NAME => 'TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:14,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d51b53151407a9b9ad82a60ad302f998, NAME => 'TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d51b53151407a9b9ad82a60ad302f998, NAME => 'TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:14,978 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:14,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40774 deadline: 1731278354973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:14,979 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:14,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40778 deadline: 1731278354973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:14,979 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:14,979 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:14,979 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:14,979 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:14,979 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:14,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40754 deadline: 1731278354973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:14,979 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:14,979 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:14,979 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:14,980 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:14,980 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:14,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40738 deadline: 1731278354973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:14,980 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:14,980 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:14,980 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:14,980 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:14,980 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:14,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40736 deadline: 1731278354973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:14,981 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:14,981 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:14,981 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:15,083 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-10T22:38:15,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. 2024-11-10T22:38:15,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. as already flushing 2024-11-10T22:38:15,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. 2024-11-10T22:38:15,084 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => d51b53151407a9b9ad82a60ad302f998, NAME => 'TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:15,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => d51b53151407a9b9ad82a60ad302f998, NAME => 'TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:15,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d51b53151407a9b9ad82a60ad302f998, NAME => 'TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d51b53151407a9b9ad82a60ad302f998, NAME => 'TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:15,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-10T22:38:15,148 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=178 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/C/a8807d8fabb1426b86517d7fbd5027cb 2024-11-10T22:38:15,154 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/A/3c53ea79ea674e708881fdc7007f15ac as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/3c53ea79ea674e708881fdc7007f15ac 2024-11-10T22:38:15,158 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/3c53ea79ea674e708881fdc7007f15ac, entries=250, sequenceid=178, filesize=47.3 K 2024-11-10T22:38:15,160 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/B/486fc2c33da9494f8fc3be85f64d8c5c as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B/486fc2c33da9494f8fc3be85f64d8c5c 2024-11-10T22:38:15,164 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B/486fc2c33da9494f8fc3be85f64d8c5c, entries=150, sequenceid=178, filesize=11.9 K 2024-11-10T22:38:15,165 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/C/a8807d8fabb1426b86517d7fbd5027cb as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/C/a8807d8fabb1426b86517d7fbd5027cb 2024-11-10T22:38:15,170 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/C/a8807d8fabb1426b86517d7fbd5027cb, entries=150, sequenceid=178, filesize=11.9 K 2024-11-10T22:38:15,171 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for d51b53151407a9b9ad82a60ad302f998 in 556ms, sequenceid=178, compaction requested=true 2024-11-10T22:38:15,171 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d51b53151407a9b9ad82a60ad302f998: 2024-11-10T22:38:15,171 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d51b53151407a9b9ad82a60ad302f998:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T22:38:15,171 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T22:38:15,171 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T22:38:15,172 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T22:38:15,172 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d51b53151407a9b9ad82a60ad302f998:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T22:38:15,172 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T22:38:15,172 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d51b53151407a9b9ad82a60ad302f998:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T22:38:15,172 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T22:38:15,173 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T22:38:15,173 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1541): d51b53151407a9b9ad82a60ad302f998/B is initiating minor compaction (all files) 2024-11-10T22:38:15,173 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d51b53151407a9b9ad82a60ad302f998/B in TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. 2024-11-10T22:38:15,173 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B/79f0d3719e57470d8fb01c738594f277, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B/c47796a3673f4344bbee360f081a9433, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B/486fc2c33da9494f8fc3be85f64d8c5c] into tmpdir=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp, totalSize=35.9 K 2024-11-10T22:38:15,173 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T22:38:15,173 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HStore(1541): d51b53151407a9b9ad82a60ad302f998/A is initiating minor compaction (all files) 2024-11-10T22:38:15,173 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d51b53151407a9b9ad82a60ad302f998/A in TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. 2024-11-10T22:38:15,173 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/8810867aca3b42af8c217923a96327aa, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/359698e468ba47788d1b67fe67724389, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/3c53ea79ea674e708881fdc7007f15ac] into tmpdir=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp, totalSize=108.3 K 2024-11-10T22:38:15,174 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. 2024-11-10T22:38:15,174 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. files: [hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/8810867aca3b42af8c217923a96327aa, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/359698e468ba47788d1b67fe67724389, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/3c53ea79ea674e708881fdc7007f15ac] 2024-11-10T22:38:15,174 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting 79f0d3719e57470d8fb01c738594f277, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1731278292948 2024-11-10T22:38:15,174 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8810867aca3b42af8c217923a96327aa, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1731278292948 2024-11-10T22:38:15,174 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting c47796a3673f4344bbee360f081a9433, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=164, earliestPutTs=1731278293616 2024-11-10T22:38:15,175 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 359698e468ba47788d1b67fe67724389, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=164, earliestPutTs=1731278293616 2024-11-10T22:38:15,175 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting 486fc2c33da9494f8fc3be85f64d8c5c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1731278293986 2024-11-10T22:38:15,175 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3c53ea79ea674e708881fdc7007f15ac, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1731278293986 2024-11-10T22:38:15,184 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d51b53151407a9b9ad82a60ad302f998] 2024-11-10T22:38:15,189 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d51b53151407a9b9ad82a60ad302f998#B#compaction#160 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T22:38:15,189 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111005b02edf4dc54a5f8a765ecd1142e085_d51b53151407a9b9ad82a60ad302f998 store=[table=TestAcidGuarantees family=A region=d51b53151407a9b9ad82a60ad302f998] 2024-11-10T22:38:15,189 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/B/83e5a514e8b04664aa895cb8f7c55ff3 is 50, key is test_row_0/B:col10/1731278294614/Put/seqid=0 2024-11-10T22:38:15,191 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111005b02edf4dc54a5f8a765ecd1142e085_d51b53151407a9b9ad82a60ad302f998, store=[table=TestAcidGuarantees family=A region=d51b53151407a9b9ad82a60ad302f998] 2024-11-10T22:38:15,192 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111005b02edf4dc54a5f8a765ecd1142e085_d51b53151407a9b9ad82a60ad302f998 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d51b53151407a9b9ad82a60ad302f998] 2024-11-10T22:38:15,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742011_1187 (size=12561) 2024-11-10T22:38:15,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742012_1188 (size=4469) 2024-11-10T22:38:15,237 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-10T22:38:15,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. 2024-11-10T22:38:15,237 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing d51b53151407a9b9ad82a60ad302f998 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-10T22:38:15,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110eb9c8c32ba5a4bd4868a3b672eeaad62_d51b53151407a9b9ad82a60ad302f998 is 50, key is test_row_0/A:col10/1731278294646/Put/seqid=0 2024-11-10T22:38:15,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(8855): Flush requested on d51b53151407a9b9ad82a60ad302f998 2024-11-10T22:38:15,284 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. as already flushing 2024-11-10T22:38:15,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742013_1189 (size=12304) 2024-11-10T22:38:15,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:15,300 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110eb9c8c32ba5a4bd4868a3b672eeaad62_d51b53151407a9b9ad82a60ad302f998 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110eb9c8c32ba5a4bd4868a3b672eeaad62_d51b53151407a9b9ad82a60ad302f998 2024-11-10T22:38:15,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/A/5bfa0d7c3e0547e3933078168f04ea8a, store: [table=TestAcidGuarantees family=A region=d51b53151407a9b9ad82a60ad302f998] 2024-11-10T22:38:15,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/A/5bfa0d7c3e0547e3933078168f04ea8a is 175, key is test_row_0/A:col10/1731278294646/Put/seqid=0 2024-11-10T22:38:15,317 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:15,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40754 deadline: 1731278355306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:15,318 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:15,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40736 deadline: 1731278355306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:15,318 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:15,318 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:15,318 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:15,318 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:15,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40774 deadline: 1731278355307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:15,318 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:15,319 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:15,319 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:15,319 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:15,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40778 deadline: 1731278355311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:15,319 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:15,319 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:15,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40738 deadline: 1731278355312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:15,319 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:15,319 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:15,319 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:15,319 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:15,319 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:15,319 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:15,319 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:15,319 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:15,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742014_1190 (size=31105) 2024-11-10T22:38:15,331 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=201, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/A/5bfa0d7c3e0547e3933078168f04ea8a 2024-11-10T22:38:15,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/B/d38e66981d7c4aba9163521b3bc9196b is 50, key is test_row_0/B:col10/1731278294646/Put/seqid=0 2024-11-10T22:38:15,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742015_1191 (size=12151) 2024-11-10T22:38:15,362 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=201 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/B/d38e66981d7c4aba9163521b3bc9196b 2024-11-10T22:38:15,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/C/1e93269280ef4763832d5c3af1f737c3 is 50, key is test_row_0/C:col10/1731278294646/Put/seqid=0 2024-11-10T22:38:15,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742016_1192 (size=12151) 2024-11-10T22:38:15,401 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=201 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/C/1e93269280ef4763832d5c3af1f737c3 2024-11-10T22:38:15,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/A/5bfa0d7c3e0547e3933078168f04ea8a as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/5bfa0d7c3e0547e3933078168f04ea8a 2024-11-10T22:38:15,417 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/5bfa0d7c3e0547e3933078168f04ea8a, entries=150, sequenceid=201, filesize=30.4 K 2024-11-10T22:38:15,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/B/d38e66981d7c4aba9163521b3bc9196b as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B/d38e66981d7c4aba9163521b3bc9196b 2024-11-10T22:38:15,424 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B/d38e66981d7c4aba9163521b3bc9196b, entries=150, sequenceid=201, filesize=11.9 K 2024-11-10T22:38:15,424 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:15,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40754 deadline: 1731278355422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:15,425 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:15,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40736 deadline: 1731278355423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:15,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/C/1e93269280ef4763832d5c3af1f737c3 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/C/1e93269280ef4763832d5c3af1f737c3 2024-11-10T22:38:15,425 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:15,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40778 deadline: 1731278355423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:15,425 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:15,426 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:15,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40738 deadline: 1731278355423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:15,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40774 deadline: 1731278355423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:15,426 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:15,426 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:15,426 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:15,426 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:15,426 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:15,426 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:15,426 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:15,426 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:15,426 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:15,426 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:15,426 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:15,426 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:15,426 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:15,427 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:15,427 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:15,430 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/C/1e93269280ef4763832d5c3af1f737c3, entries=150, sequenceid=201, filesize=11.9 K 2024-11-10T22:38:15,431 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for d51b53151407a9b9ad82a60ad302f998 in 194ms, sequenceid=201, compaction requested=true 2024-11-10T22:38:15,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for d51b53151407a9b9ad82a60ad302f998: 2024-11-10T22:38:15,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. 2024-11-10T22:38:15,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-11-10T22:38:15,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-11-10T22:38:15,437 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-11-10T22:38:15,437 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4270 sec 2024-11-10T22:38:15,440 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 1.4340 sec 2024-11-10T22:38:15,610 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d51b53151407a9b9ad82a60ad302f998#A#compaction#159 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T22:38:15,611 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/A/4c31db80c94b46a18023ecf1c2adf870 is 175, key is test_row_0/A:col10/1731278294614/Put/seqid=0 2024-11-10T22:38:15,620 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/B/83e5a514e8b04664aa895cb8f7c55ff3 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B/83e5a514e8b04664aa895cb8f7c55ff3 2024-11-10T22:38:15,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742017_1193 (size=31515) 2024-11-10T22:38:15,628 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d51b53151407a9b9ad82a60ad302f998/B of d51b53151407a9b9ad82a60ad302f998 into 83e5a514e8b04664aa895cb8f7c55ff3(size=12.3 K), total size for store is 24.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T22:38:15,628 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d51b53151407a9b9ad82a60ad302f998: 2024-11-10T22:38:15,629 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., storeName=d51b53151407a9b9ad82a60ad302f998/B, priority=13, startTime=1731278295171; duration=0sec 2024-11-10T22:38:15,630 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T22:38:15,630 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d51b53151407a9b9ad82a60ad302f998:B 2024-11-10T22:38:15,630 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T22:38:15,632 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/A/4c31db80c94b46a18023ecf1c2adf870 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/4c31db80c94b46a18023ecf1c2adf870 2024-11-10T22:38:15,633 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48862 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T22:38:15,635 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1541): d51b53151407a9b9ad82a60ad302f998/C is initiating minor compaction (all files) 2024-11-10T22:38:15,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(8855): Flush requested on d51b53151407a9b9ad82a60ad302f998 2024-11-10T22:38:15,636 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T22:38:15,636 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d51b53151407a9b9ad82a60ad302f998/C in TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. 2024-11-10T22:38:15,636 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d51b53151407a9b9ad82a60ad302f998 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-10T22:38:15,636 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/C/35bee95a01c640aeb8ac8cd0c20c0b9e, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/C/95e6a53bc8fc48d198df5c48e16ede98, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/C/a8807d8fabb1426b86517d7fbd5027cb, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/C/1e93269280ef4763832d5c3af1f737c3] into tmpdir=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp, totalSize=47.7 K 2024-11-10T22:38:15,638 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting 35bee95a01c640aeb8ac8cd0c20c0b9e, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1731278292948 2024-11-10T22:38:15,640 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting 95e6a53bc8fc48d198df5c48e16ede98, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=164, earliestPutTs=1731278293616 2024-11-10T22:38:15,641 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting a8807d8fabb1426b86517d7fbd5027cb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1731278293986 2024-11-10T22:38:15,641 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d51b53151407a9b9ad82a60ad302f998/A of d51b53151407a9b9ad82a60ad302f998 into 4c31db80c94b46a18023ecf1c2adf870(size=30.8 K), total size for store is 61.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T22:38:15,641 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d51b53151407a9b9ad82a60ad302f998: 2024-11-10T22:38:15,641 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., storeName=d51b53151407a9b9ad82a60ad302f998/A, priority=13, startTime=1731278295171; duration=0sec 2024-11-10T22:38:15,642 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T22:38:15,642 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d51b53151407a9b9ad82a60ad302f998:A 2024-11-10T22:38:15,643 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting 1e93269280ef4763832d5c3af1f737c3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1731278294646 2024-11-10T22:38:15,653 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411109ede2dc7094f426386b4ade0bd0f7646_d51b53151407a9b9ad82a60ad302f998 is 50, key is test_row_0/A:col10/1731278295309/Put/seqid=0 2024-11-10T22:38:15,665 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d51b53151407a9b9ad82a60ad302f998#C#compaction#165 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T22:38:15,666 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/C/a5fdb2af495045549dce194f5b74138a is 50, key is test_row_0/C:col10/1731278294646/Put/seqid=0 2024-11-10T22:38:15,675 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:15,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40774 deadline: 1731278355667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:15,676 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:15,676 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:15,676 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:15,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742018_1194 (size=17284) 2024-11-10T22:38:15,678 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:15,680 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:15,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40754 deadline: 1731278355668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:15,680 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:15,680 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:15,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40736 deadline: 1731278355669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:15,680 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:15,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40778 deadline: 1731278355670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:15,681 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:15,681 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:15,681 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:15,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40738 deadline: 1731278355670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:15,681 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:15,681 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:15,681 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:15,683 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:15,683 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:15,683 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:15,684 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411109ede2dc7094f426386b4ade0bd0f7646_d51b53151407a9b9ad82a60ad302f998 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411109ede2dc7094f426386b4ade0bd0f7646_d51b53151407a9b9ad82a60ad302f998 2024-11-10T22:38:15,684 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:15,684 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:15,684 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:15,687 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/A/3b7faa6de25f4d35bd1ce967d3f2d017, store: [table=TestAcidGuarantees family=A region=d51b53151407a9b9ad82a60ad302f998] 2024-11-10T22:38:15,688 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/A/3b7faa6de25f4d35bd1ce967d3f2d017 is 175, key is test_row_0/A:col10/1731278295309/Put/seqid=0 2024-11-10T22:38:15,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742019_1195 (size=12595) 2024-11-10T22:38:15,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742020_1196 (size=48389) 2024-11-10T22:38:15,790 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:15,790 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:15,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40774 deadline: 1731278355782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:15,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40754 deadline: 1731278355783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:15,791 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:15,791 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:15,791 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:15,791 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:15,791 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:15,791 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:15,792 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:15,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40738 deadline: 1731278355792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:15,793 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:15,793 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:15,793 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:15,793 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:15,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40778 deadline: 1731278355792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:15,793 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:15,793 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:15,793 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:15,793 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:15,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40736 deadline: 1731278355792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:15,793 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:15,793 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:15,793 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:16,006 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:16,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40754 deadline: 1731278356002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:16,007 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:16,007 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:16,007 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:16,007 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:16,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40774 deadline: 1731278356002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:16,007 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:16,007 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:16,007 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:16,007 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:16,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40738 deadline: 1731278356002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:16,008 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:16,008 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:16,008 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:16,008 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:16,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40736 deadline: 1731278356002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:16,008 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:16,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40778 deadline: 1731278356002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:16,008 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:16,008 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:16,008 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:16,008 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:16,009 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:16,009 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:16,099 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=218, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/A/3b7faa6de25f4d35bd1ce967d3f2d017 2024-11-10T22:38:16,100 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/C/a5fdb2af495045549dce194f5b74138a as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/C/a5fdb2af495045549dce194f5b74138a 2024-11-10T22:38:16,108 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/B/34549fb4911140f0bbd6df8c7ea97cc0 is 50, key is test_row_0/B:col10/1731278295309/Put/seqid=0 2024-11-10T22:38:16,109 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d51b53151407a9b9ad82a60ad302f998/C of d51b53151407a9b9ad82a60ad302f998 into a5fdb2af495045549dce194f5b74138a(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T22:38:16,109 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d51b53151407a9b9ad82a60ad302f998: 2024-11-10T22:38:16,109 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., storeName=d51b53151407a9b9ad82a60ad302f998/C, priority=12, startTime=1731278295172; duration=0sec 2024-11-10T22:38:16,109 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T22:38:16,109 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d51b53151407a9b9ad82a60ad302f998:C 2024-11-10T22:38:16,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742021_1197 (size=12151) 2024-11-10T22:38:16,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-10T22:38:16,153 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T22:38:16,153 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T22:38:16,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-11-10T22:38:16,156 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T22:38:16,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-10T22:38:16,156 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T22:38:16,156 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T22:38:16,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-10T22:38:16,309 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-10T22:38:16,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. 2024-11-10T22:38:16,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. as already flushing 2024-11-10T22:38:16,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. 2024-11-10T22:38:16,309 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => d51b53151407a9b9ad82a60ad302f998, NAME => 'TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:16,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => d51b53151407a9b9ad82a60ad302f998, NAME => 'TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:16,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d51b53151407a9b9ad82a60ad302f998, NAME => 'TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d51b53151407a9b9ad82a60ad302f998, NAME => 'TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:16,313 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:16,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40754 deadline: 1731278356312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:16,313 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:16,313 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:16,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40736 deadline: 1731278356312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:16,313 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:16,313 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:16,314 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:16,314 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:16,314 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:16,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40738 deadline: 1731278356312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:16,314 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:16,314 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:16,314 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:16,314 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:16,318 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:16,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40774 deadline: 1731278356316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:16,319 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:16,319 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:16,319 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:16,323 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:16,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40778 deadline: 1731278356323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:16,324 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:16,324 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:16,324 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:16,462 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-10T22:38:16,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. 2024-11-10T22:38:16,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. as already flushing 2024-11-10T22:38:16,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. 2024-11-10T22:38:16,463 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => d51b53151407a9b9ad82a60ad302f998, NAME => 'TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:16,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => d51b53151407a9b9ad82a60ad302f998, NAME => 'TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:16,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d51b53151407a9b9ad82a60ad302f998, NAME => 'TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d51b53151407a9b9ad82a60ad302f998, NAME => 'TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:16,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-10T22:38:16,518 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=218 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/B/34549fb4911140f0bbd6df8c7ea97cc0 2024-11-10T22:38:16,527 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/C/eac6ef3171ea4386a4af1c38e482078f is 50, key is test_row_0/C:col10/1731278295309/Put/seqid=0 2024-11-10T22:38:16,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742022_1198 (size=12151) 2024-11-10T22:38:16,616 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-10T22:38:16,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. 2024-11-10T22:38:16,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. as already flushing 2024-11-10T22:38:16,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. 2024-11-10T22:38:16,616 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => d51b53151407a9b9ad82a60ad302f998, NAME => 'TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:16,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => d51b53151407a9b9ad82a60ad302f998, NAME => 'TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:16,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d51b53151407a9b9ad82a60ad302f998, NAME => 'TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d51b53151407a9b9ad82a60ad302f998, NAME => 'TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:16,769 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-10T22:38:16,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. 2024-11-10T22:38:16,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. as already flushing 2024-11-10T22:38:16,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. 2024-11-10T22:38:16,770 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => d51b53151407a9b9ad82a60ad302f998, NAME => 'TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:16,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => d51b53151407a9b9ad82a60ad302f998, NAME => 'TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:16,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d51b53151407a9b9ad82a60ad302f998, NAME => 'TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d51b53151407a9b9ad82a60ad302f998, NAME => 'TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:16,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-10T22:38:16,825 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:16,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40754 deadline: 1731278356822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:16,826 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:16,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40736 deadline: 1731278356823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:16,826 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:16,826 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:16,826 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:16,826 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:16,826 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:16,826 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:16,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40738 deadline: 1731278356823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:16,826 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:16,827 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:16,827 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:16,827 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:16,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40774 deadline: 1731278356823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:16,827 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:16,827 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:16,827 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:16,827 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:16,836 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:16,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40778 deadline: 1731278356832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:16,837 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:16,837 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:16,837 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:16,923 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-10T22:38:16,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. 2024-11-10T22:38:16,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. as already flushing 2024-11-10T22:38:16,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. 2024-11-10T22:38:16,924 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => d51b53151407a9b9ad82a60ad302f998, NAME => 'TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:16,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => d51b53151407a9b9ad82a60ad302f998, NAME => 'TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:16,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d51b53151407a9b9ad82a60ad302f998, NAME => 'TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d51b53151407a9b9ad82a60ad302f998, NAME => 'TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:16,931 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=218 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/C/eac6ef3171ea4386a4af1c38e482078f 2024-11-10T22:38:16,939 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/A/3b7faa6de25f4d35bd1ce967d3f2d017 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/3b7faa6de25f4d35bd1ce967d3f2d017 2024-11-10T22:38:16,944 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/3b7faa6de25f4d35bd1ce967d3f2d017, entries=250, sequenceid=218, filesize=47.3 K 2024-11-10T22:38:16,946 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/B/34549fb4911140f0bbd6df8c7ea97cc0 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B/34549fb4911140f0bbd6df8c7ea97cc0 2024-11-10T22:38:16,951 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B/34549fb4911140f0bbd6df8c7ea97cc0, entries=150, sequenceid=218, filesize=11.9 K 2024-11-10T22:38:16,952 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/C/eac6ef3171ea4386a4af1c38e482078f as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/C/eac6ef3171ea4386a4af1c38e482078f 2024-11-10T22:38:16,957 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/C/eac6ef3171ea4386a4af1c38e482078f, entries=150, sequenceid=218, filesize=11.9 K 2024-11-10T22:38:16,958 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for d51b53151407a9b9ad82a60ad302f998 in 1321ms, sequenceid=218, compaction requested=true 2024-11-10T22:38:16,958 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d51b53151407a9b9ad82a60ad302f998: 2024-11-10T22:38:16,958 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d51b53151407a9b9ad82a60ad302f998:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T22:38:16,958 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T22:38:16,958 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T22:38:16,958 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d51b53151407a9b9ad82a60ad302f998:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T22:38:16,958 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T22:38:16,958 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d51b53151407a9b9ad82a60ad302f998:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T22:38:16,958 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T22:38:16,958 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T22:38:16,960 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111009 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T22:38:16,960 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HStore(1541): d51b53151407a9b9ad82a60ad302f998/A is initiating minor compaction (all files) 2024-11-10T22:38:16,960 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d51b53151407a9b9ad82a60ad302f998/A in TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. 2024-11-10T22:38:16,960 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/4c31db80c94b46a18023ecf1c2adf870, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/5bfa0d7c3e0547e3933078168f04ea8a, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/3b7faa6de25f4d35bd1ce967d3f2d017] into tmpdir=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp, totalSize=108.4 K 2024-11-10T22:38:16,960 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. 2024-11-10T22:38:16,960 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. files: [hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/4c31db80c94b46a18023ecf1c2adf870, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/5bfa0d7c3e0547e3933078168f04ea8a, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/3b7faa6de25f4d35bd1ce967d3f2d017] 2024-11-10T22:38:16,960 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T22:38:16,960 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1541): d51b53151407a9b9ad82a60ad302f998/B is initiating minor compaction (all files) 2024-11-10T22:38:16,960 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d51b53151407a9b9ad82a60ad302f998/B in TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. 2024-11-10T22:38:16,960 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B/83e5a514e8b04664aa895cb8f7c55ff3, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B/d38e66981d7c4aba9163521b3bc9196b, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B/34549fb4911140f0bbd6df8c7ea97cc0] into tmpdir=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp, totalSize=36.0 K 2024-11-10T22:38:16,961 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4c31db80c94b46a18023ecf1c2adf870, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1731278293986 2024-11-10T22:38:16,961 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting 83e5a514e8b04664aa895cb8f7c55ff3, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1731278293986 2024-11-10T22:38:16,962 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5bfa0d7c3e0547e3933078168f04ea8a, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1731278294646 2024-11-10T22:38:16,962 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting d38e66981d7c4aba9163521b3bc9196b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1731278294646 2024-11-10T22:38:16,962 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3b7faa6de25f4d35bd1ce967d3f2d017, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1731278295303 2024-11-10T22:38:16,962 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting 34549fb4911140f0bbd6df8c7ea97cc0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1731278295303 2024-11-10T22:38:16,971 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d51b53151407a9b9ad82a60ad302f998] 2024-11-10T22:38:16,975 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241110f6b05dd27fa5466ea7170e7652964736_d51b53151407a9b9ad82a60ad302f998 store=[table=TestAcidGuarantees family=A region=d51b53151407a9b9ad82a60ad302f998] 2024-11-10T22:38:16,975 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d51b53151407a9b9ad82a60ad302f998#B#compaction#169 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T22:38:16,976 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/B/561a785c66bc46d795bd14b7071fd4ab is 50, key is test_row_0/B:col10/1731278295309/Put/seqid=0 2024-11-10T22:38:16,977 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241110f6b05dd27fa5466ea7170e7652964736_d51b53151407a9b9ad82a60ad302f998, store=[table=TestAcidGuarantees family=A region=d51b53151407a9b9ad82a60ad302f998] 2024-11-10T22:38:16,977 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110f6b05dd27fa5466ea7170e7652964736_d51b53151407a9b9ad82a60ad302f998 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d51b53151407a9b9ad82a60ad302f998] 2024-11-10T22:38:16,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742023_1199 (size=12663) 2024-11-10T22:38:16,997 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/B/561a785c66bc46d795bd14b7071fd4ab as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B/561a785c66bc46d795bd14b7071fd4ab 2024-11-10T22:38:17,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742024_1200 (size=4469) 2024-11-10T22:38:17,005 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d51b53151407a9b9ad82a60ad302f998/B of d51b53151407a9b9ad82a60ad302f998 into 561a785c66bc46d795bd14b7071fd4ab(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T22:38:17,005 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d51b53151407a9b9ad82a60ad302f998: 2024-11-10T22:38:17,005 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., storeName=d51b53151407a9b9ad82a60ad302f998/B, priority=13, startTime=1731278296958; duration=0sec 2024-11-10T22:38:17,005 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T22:38:17,005 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d51b53151407a9b9ad82a60ad302f998:B 2024-11-10T22:38:17,005 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-10T22:38:17,006 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-10T22:38:17,007 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-10T22:38:17,007 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. because compaction request was cancelled 2024-11-10T22:38:17,007 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d51b53151407a9b9ad82a60ad302f998:C 2024-11-10T22:38:17,009 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d51b53151407a9b9ad82a60ad302f998#A#compaction#168 average throughput is 0.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T22:38:17,010 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/A/1e4c9970088f487d8bee0e437576b116 is 175, key is test_row_0/A:col10/1731278295309/Put/seqid=0 2024-11-10T22:38:17,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742025_1201 (size=31617) 2024-11-10T22:38:17,026 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/A/1e4c9970088f487d8bee0e437576b116 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/1e4c9970088f487d8bee0e437576b116 2024-11-10T22:38:17,034 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d51b53151407a9b9ad82a60ad302f998/A of d51b53151407a9b9ad82a60ad302f998 into 1e4c9970088f487d8bee0e437576b116(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T22:38:17,034 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d51b53151407a9b9ad82a60ad302f998: 2024-11-10T22:38:17,034 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., storeName=d51b53151407a9b9ad82a60ad302f998/A, priority=13, startTime=1731278296958; duration=0sec 2024-11-10T22:38:17,035 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T22:38:17,035 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d51b53151407a9b9ad82a60ad302f998:A 2024-11-10T22:38:17,075 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-10T22:38:17,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. 2024-11-10T22:38:17,076 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing d51b53151407a9b9ad82a60ad302f998 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-10T22:38:17,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111021d6e25057084a1baaf954c74667b713_d51b53151407a9b9ad82a60ad302f998 is 50, key is test_row_0/A:col10/1731278295667/Put/seqid=0 2024-11-10T22:38:17,095 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742026_1202 (size=12304) 2024-11-10T22:38:17,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-10T22:38:17,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:17,502 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111021d6e25057084a1baaf954c74667b713_d51b53151407a9b9ad82a60ad302f998 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111021d6e25057084a1baaf954c74667b713_d51b53151407a9b9ad82a60ad302f998 2024-11-10T22:38:17,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/A/34133856ee764d95ab9341c48b9ee49b, store: [table=TestAcidGuarantees family=A region=d51b53151407a9b9ad82a60ad302f998] 2024-11-10T22:38:17,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/A/34133856ee764d95ab9341c48b9ee49b is 175, key is test_row_0/A:col10/1731278295667/Put/seqid=0 2024-11-10T22:38:17,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742027_1203 (size=31105) 2024-11-10T22:38:17,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(8855): Flush requested on d51b53151407a9b9ad82a60ad302f998 2024-11-10T22:38:17,836 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. as already flushing 2024-11-10T22:38:17,855 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:17,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40774 deadline: 1731278357851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:17,856 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:17,856 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:17,856 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:17,856 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:17,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40754 deadline: 1731278357852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:17,856 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:17,856 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:17,856 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:17,856 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:17,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40778 deadline: 1731278357852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:17,857 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:17,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40736 deadline: 1731278357854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:17,857 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:17,857 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:17,857 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:17,857 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 849d0d71310f,43753,1731278250663 for 'test_row_0' in d51b53151407a9b9ad82a60ad302f998 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2187 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-10T22:38:17,857 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:17,858 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:17,858 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:17,858 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:17,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40738 deadline: 1731278357855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:17,859 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:17,859 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:17,859 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:17,908 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=242, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/A/34133856ee764d95ab9341c48b9ee49b 2024-11-10T22:38:17,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/B/d7c0f2f7d5e541a5a942cd011800c1bb is 50, key is test_row_0/B:col10/1731278295667/Put/seqid=0 2024-11-10T22:38:17,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742028_1204 (size=12151) 2024-11-10T22:38:17,964 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:17,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40774 deadline: 1731278357962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:17,965 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:17,965 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:17,965 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:17,965 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:17,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40754 deadline: 1731278357962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:17,965 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:17,965 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:17,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40736 deadline: 1731278357963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:17,965 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:17,965 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:17,966 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:17,966 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:17,966 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:17,966 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:17,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40738 deadline: 1731278357963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:17,966 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:17,966 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:17,966 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:18,175 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:18,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40774 deadline: 1731278358172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:18,176 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:18,176 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:18,176 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:18,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40754 deadline: 1731278358173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:18,176 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:18,176 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:18,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40736 deadline: 1731278358173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:18,176 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:18,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40738 deadline: 1731278358173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:18,176 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:18,176 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:18,176 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:18,176 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:18,176 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:18,177 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:18,177 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:18,177 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:18,177 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:18,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-10T22:38:18,330 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=242 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/B/d7c0f2f7d5e541a5a942cd011800c1bb 2024-11-10T22:38:18,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/C/32e27ec7ee7248ff883dbb33206693ca is 50, key is test_row_0/C:col10/1731278295667/Put/seqid=0 2024-11-10T22:38:18,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742029_1205 (size=12151) 2024-11-10T22:38:18,343 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=242 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/C/32e27ec7ee7248ff883dbb33206693ca 2024-11-10T22:38:18,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/A/34133856ee764d95ab9341c48b9ee49b as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/34133856ee764d95ab9341c48b9ee49b 2024-11-10T22:38:18,352 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/34133856ee764d95ab9341c48b9ee49b, entries=150, sequenceid=242, filesize=30.4 K 2024-11-10T22:38:18,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/B/d7c0f2f7d5e541a5a942cd011800c1bb as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B/d7c0f2f7d5e541a5a942cd011800c1bb 2024-11-10T22:38:18,358 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B/d7c0f2f7d5e541a5a942cd011800c1bb, entries=150, sequenceid=242, filesize=11.9 K 2024-11-10T22:38:18,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/C/32e27ec7ee7248ff883dbb33206693ca as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/C/32e27ec7ee7248ff883dbb33206693ca 2024-11-10T22:38:18,363 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/C/32e27ec7ee7248ff883dbb33206693ca, entries=150, sequenceid=242, filesize=11.9 K 2024-11-10T22:38:18,364 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for d51b53151407a9b9ad82a60ad302f998 in 1288ms, sequenceid=242, compaction requested=true 2024-11-10T22:38:18,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for d51b53151407a9b9ad82a60ad302f998: 2024-11-10T22:38:18,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. 2024-11-10T22:38:18,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-11-10T22:38:18,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-11-10T22:38:18,368 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-11-10T22:38:18,368 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2100 sec 2024-11-10T22:38:18,370 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 2.2160 sec 2024-11-10T22:38:18,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(8855): Flush requested on d51b53151407a9b9ad82a60ad302f998 2024-11-10T22:38:18,493 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T22:38:18,493 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d51b53151407a9b9ad82a60ad302f998 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-10T22:38:18,520 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411106ff5994cbf9a454c90b982bd4f09d554_d51b53151407a9b9ad82a60ad302f998 is 50, key is test_row_0/A:col10/1731278297839/Put/seqid=0 2024-11-10T22:38:18,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742030_1206 (size=12404) 2024-11-10T22:38:18,526 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:18,530 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411106ff5994cbf9a454c90b982bd4f09d554_d51b53151407a9b9ad82a60ad302f998 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411106ff5994cbf9a454c90b982bd4f09d554_d51b53151407a9b9ad82a60ad302f998 2024-11-10T22:38:18,531 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/A/05125de77e6540978f99ea1243b2d47c, store: [table=TestAcidGuarantees family=A region=d51b53151407a9b9ad82a60ad302f998] 2024-11-10T22:38:18,531 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/A/05125de77e6540978f99ea1243b2d47c is 175, key is test_row_0/A:col10/1731278297839/Put/seqid=0 2024-11-10T22:38:18,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742031_1207 (size=31205) 2024-11-10T22:38:18,536 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:18,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40754 deadline: 1731278358532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:18,536 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:18,536 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:18,536 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:18,536 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:18,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40738 deadline: 1731278358534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:18,537 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:18,537 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:18,537 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:18,537 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:18,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40736 deadline: 1731278358535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:18,537 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:18,537 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:18,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40774 deadline: 1731278358536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:18,537 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:18,537 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:18,537 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:18,537 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:18,538 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:18,647 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:18,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40754 deadline: 1731278358642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:18,647 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:18,647 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:18,647 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:18,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40738 deadline: 1731278358642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:18,647 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:18,647 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:18,648 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:18,648 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:18,648 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:18,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40736 deadline: 1731278358642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:18,648 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:18,648 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:18,648 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:18,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40774 deadline: 1731278358642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:18,648 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:18,648 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:18,648 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:18,648 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:18,855 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:18,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40754 deadline: 1731278358852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:18,855 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:18,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40738 deadline: 1731278358852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:18,855 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:18,855 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:18,856 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:18,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40736 deadline: 1731278358852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:18,856 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:18,856 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:18,856 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:18,856 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:18,856 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:18,856 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:18,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40774 deadline: 1731278358852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:18,856 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:18,856 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:18,856 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:18,856 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:18,856 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:18,936 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=262, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/A/05125de77e6540978f99ea1243b2d47c 2024-11-10T22:38:18,945 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/B/d01eb6c178c64462b69c963b493841bb is 50, key is test_row_0/B:col10/1731278297839/Put/seqid=0 2024-11-10T22:38:18,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742032_1208 (size=12251) 2024-11-10T22:38:19,166 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:19,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40754 deadline: 1731278359163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:19,167 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:19,167 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:19,167 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:19,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40738 deadline: 1731278359163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:19,167 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:19,167 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:19,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40774 deadline: 1731278359163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:19,167 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:19,167 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:19,167 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:19,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40736 deadline: 1731278359163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:19,167 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:19,167 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:19,167 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:19,167 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:19,167 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:19,168 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:19,168 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:19,350 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=262 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/B/d01eb6c178c64462b69c963b493841bb 2024-11-10T22:38:19,359 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/C/2050541d722e42fd9ae6227e2b1e4400 is 50, key is test_row_0/C:col10/1731278297839/Put/seqid=0 2024-11-10T22:38:19,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742033_1209 (size=12251) 2024-11-10T22:38:19,673 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:19,673 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:19,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40754 deadline: 1731278359673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:19,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40738 deadline: 1731278359673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:19,674 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:19,674 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:19,674 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:19,674 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:19,674 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:19,674 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:19,674 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:19,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40736 deadline: 1731278359673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:19,674 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:19,674 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:19,674 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:19,687 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:19,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40774 deadline: 1731278359682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:19,688 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:19,688 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:19,688 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:19,767 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=262 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/C/2050541d722e42fd9ae6227e2b1e4400 2024-11-10T22:38:19,773 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/A/05125de77e6540978f99ea1243b2d47c as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/05125de77e6540978f99ea1243b2d47c 2024-11-10T22:38:19,777 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/05125de77e6540978f99ea1243b2d47c, entries=150, sequenceid=262, filesize=30.5 K 2024-11-10T22:38:19,778 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/B/d01eb6c178c64462b69c963b493841bb as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B/d01eb6c178c64462b69c963b493841bb 2024-11-10T22:38:19,783 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B/d01eb6c178c64462b69c963b493841bb, entries=150, sequenceid=262, filesize=12.0 K 2024-11-10T22:38:19,784 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/C/2050541d722e42fd9ae6227e2b1e4400 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/C/2050541d722e42fd9ae6227e2b1e4400 2024-11-10T22:38:19,788 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/C/2050541d722e42fd9ae6227e2b1e4400, entries=150, sequenceid=262, filesize=12.0 K 2024-11-10T22:38:19,789 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for d51b53151407a9b9ad82a60ad302f998 in 1296ms, sequenceid=262, compaction requested=true 2024-11-10T22:38:19,789 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d51b53151407a9b9ad82a60ad302f998: 2024-11-10T22:38:19,790 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d51b53151407a9b9ad82a60ad302f998:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T22:38:19,790 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T22:38:19,790 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T22:38:19,790 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d51b53151407a9b9ad82a60ad302f998:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T22:38:19,790 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T22:38:19,790 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d51b53151407a9b9ad82a60ad302f998:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T22:38:19,790 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T22:38:19,790 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T22:38:19,791 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37065 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T22:38:19,791 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1541): d51b53151407a9b9ad82a60ad302f998/B is initiating minor compaction (all files) 2024-11-10T22:38:19,791 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d51b53151407a9b9ad82a60ad302f998/B in TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. 2024-11-10T22:38:19,791 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B/561a785c66bc46d795bd14b7071fd4ab, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B/d7c0f2f7d5e541a5a942cd011800c1bb, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B/d01eb6c178c64462b69c963b493841bb] into tmpdir=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp, totalSize=36.2 K 2024-11-10T22:38:19,792 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93927 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T22:38:19,792 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HStore(1541): d51b53151407a9b9ad82a60ad302f998/A is initiating minor compaction (all files) 2024-11-10T22:38:19,792 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting 561a785c66bc46d795bd14b7071fd4ab, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1731278295303 2024-11-10T22:38:19,792 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d51b53151407a9b9ad82a60ad302f998/A in TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. 2024-11-10T22:38:19,792 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/1e4c9970088f487d8bee0e437576b116, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/34133856ee764d95ab9341c48b9ee49b, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/05125de77e6540978f99ea1243b2d47c] into tmpdir=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp, totalSize=91.7 K 2024-11-10T22:38:19,792 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. 2024-11-10T22:38:19,792 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. files: [hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/1e4c9970088f487d8bee0e437576b116, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/34133856ee764d95ab9341c48b9ee49b, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/05125de77e6540978f99ea1243b2d47c] 2024-11-10T22:38:19,792 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting d7c0f2f7d5e541a5a942cd011800c1bb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1731278295661 2024-11-10T22:38:19,792 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1e4c9970088f487d8bee0e437576b116, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1731278295303 2024-11-10T22:38:19,793 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting d01eb6c178c64462b69c963b493841bb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1731278297839 2024-11-10T22:38:19,793 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 34133856ee764d95ab9341c48b9ee49b, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1731278295661 2024-11-10T22:38:19,794 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 05125de77e6540978f99ea1243b2d47c, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1731278297839 2024-11-10T22:38:19,801 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d51b53151407a9b9ad82a60ad302f998] 2024-11-10T22:38:19,802 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d51b53151407a9b9ad82a60ad302f998#B#compaction#176 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T22:38:19,803 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/B/6176c8a97b074e45aeb42701dd1e9ec4 is 50, key is test_row_0/B:col10/1731278297839/Put/seqid=0 2024-11-10T22:38:19,803 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411105ef2490db0894d30b7e8d53f106c5614_d51b53151407a9b9ad82a60ad302f998 store=[table=TestAcidGuarantees family=A region=d51b53151407a9b9ad82a60ad302f998] 2024-11-10T22:38:19,805 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411105ef2490db0894d30b7e8d53f106c5614_d51b53151407a9b9ad82a60ad302f998, store=[table=TestAcidGuarantees family=A region=d51b53151407a9b9ad82a60ad302f998] 2024-11-10T22:38:19,805 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411105ef2490db0894d30b7e8d53f106c5614_d51b53151407a9b9ad82a60ad302f998 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d51b53151407a9b9ad82a60ad302f998] 2024-11-10T22:38:19,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742034_1210 (size=12865) 2024-11-10T22:38:19,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742035_1211 (size=4469) 2024-11-10T22:38:19,812 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d51b53151407a9b9ad82a60ad302f998#A#compaction#177 average throughput is 2.22 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T22:38:19,813 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/A/8c0efc7b662e40fb8f682f6cb9fcd6ff is 175, key is test_row_0/A:col10/1731278297839/Put/seqid=0 2024-11-10T22:38:19,816 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/B/6176c8a97b074e45aeb42701dd1e9ec4 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B/6176c8a97b074e45aeb42701dd1e9ec4 2024-11-10T22:38:19,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742036_1212 (size=31819) 2024-11-10T22:38:19,822 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d51b53151407a9b9ad82a60ad302f998/B of d51b53151407a9b9ad82a60ad302f998 into 6176c8a97b074e45aeb42701dd1e9ec4(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T22:38:19,822 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d51b53151407a9b9ad82a60ad302f998: 2024-11-10T22:38:19,822 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., storeName=d51b53151407a9b9ad82a60ad302f998/B, priority=13, startTime=1731278299790; duration=0sec 2024-11-10T22:38:19,822 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T22:38:19,822 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d51b53151407a9b9ad82a60ad302f998:B 2024-11-10T22:38:19,822 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T22:38:19,824 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49148 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T22:38:19,824 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1541): d51b53151407a9b9ad82a60ad302f998/C is initiating minor compaction (all files) 2024-11-10T22:38:19,824 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d51b53151407a9b9ad82a60ad302f998/C in TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. 2024-11-10T22:38:19,824 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/C/a5fdb2af495045549dce194f5b74138a, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/C/eac6ef3171ea4386a4af1c38e482078f, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/C/32e27ec7ee7248ff883dbb33206693ca, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/C/2050541d722e42fd9ae6227e2b1e4400] into tmpdir=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp, totalSize=48.0 K 2024-11-10T22:38:19,825 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/A/8c0efc7b662e40fb8f682f6cb9fcd6ff as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/8c0efc7b662e40fb8f682f6cb9fcd6ff 2024-11-10T22:38:19,825 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting a5fdb2af495045549dce194f5b74138a, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1731278294646 2024-11-10T22:38:19,826 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting eac6ef3171ea4386a4af1c38e482078f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1731278295303 2024-11-10T22:38:19,826 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting 32e27ec7ee7248ff883dbb33206693ca, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1731278295661 2024-11-10T22:38:19,827 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting 2050541d722e42fd9ae6227e2b1e4400, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1731278297839 2024-11-10T22:38:19,831 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d51b53151407a9b9ad82a60ad302f998/A of d51b53151407a9b9ad82a60ad302f998 into 8c0efc7b662e40fb8f682f6cb9fcd6ff(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T22:38:19,831 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d51b53151407a9b9ad82a60ad302f998: 2024-11-10T22:38:19,831 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., storeName=d51b53151407a9b9ad82a60ad302f998/A, priority=13, startTime=1731278299789; duration=0sec 2024-11-10T22:38:19,831 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T22:38:19,832 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d51b53151407a9b9ad82a60ad302f998:A 2024-11-10T22:38:19,838 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d51b53151407a9b9ad82a60ad302f998#C#compaction#178 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T22:38:19,838 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/C/63304d55ae4e4fafaa483faac1592978 is 50, key is test_row_0/C:col10/1731278297839/Put/seqid=0 2024-11-10T22:38:19,845 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742037_1213 (size=12831) 2024-11-10T22:38:19,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(8855): Flush requested on d51b53151407a9b9ad82a60ad302f998 2024-11-10T22:38:19,876 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T22:38:19,876 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d51b53151407a9b9ad82a60ad302f998 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-10T22:38:19,886 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411101e49ef8c47474d21b1c7623a66f6d645_d51b53151407a9b9ad82a60ad302f998 is 50, key is test_row_0/A:col10/1731278299875/Put/seqid=0 2024-11-10T22:38:19,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742038_1214 (size=14994) 2024-11-10T22:38:19,949 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:19,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40778 deadline: 1731278359944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:19,949 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:19,950 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:19,950 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:20,053 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:20,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40778 deadline: 1731278360052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:20,054 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:20,054 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:20,054 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:20,251 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/C/63304d55ae4e4fafaa483faac1592978 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/C/63304d55ae4e4fafaa483faac1592978 2024-11-10T22:38:20,257 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d51b53151407a9b9ad82a60ad302f998/C of d51b53151407a9b9ad82a60ad302f998 into 63304d55ae4e4fafaa483faac1592978(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T22:38:20,257 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d51b53151407a9b9ad82a60ad302f998: 2024-11-10T22:38:20,257 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., storeName=d51b53151407a9b9ad82a60ad302f998/C, priority=12, startTime=1731278299790; duration=0sec 2024-11-10T22:38:20,257 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T22:38:20,258 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d51b53151407a9b9ad82a60ad302f998:C 2024-11-10T22:38:20,263 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:20,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40778 deadline: 1731278360262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:20,263 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:20,263 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:20,263 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:20,294 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:20,299 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411101e49ef8c47474d21b1c7623a66f6d645_d51b53151407a9b9ad82a60ad302f998 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411101e49ef8c47474d21b1c7623a66f6d645_d51b53151407a9b9ad82a60ad302f998 2024-11-10T22:38:20,300 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/A/788648d6317e475d9691263443b8e150, store: [table=TestAcidGuarantees family=A region=d51b53151407a9b9ad82a60ad302f998] 2024-11-10T22:38:20,300 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/A/788648d6317e475d9691263443b8e150 is 175, key is test_row_0/A:col10/1731278299875/Put/seqid=0 2024-11-10T22:38:20,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742039_1215 (size=39949) 2024-11-10T22:38:20,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-10T22:38:20,322 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T22:38:20,323 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T22:38:20,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2024-11-10T22:38:20,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-10T22:38:20,325 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T22:38:20,325 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T22:38:20,325 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T22:38:20,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-10T22:38:20,478 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-10T22:38:20,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. 2024-11-10T22:38:20,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. as already flushing 2024-11-10T22:38:20,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. 2024-11-10T22:38:20,478 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => d51b53151407a9b9ad82a60ad302f998, NAME => 'TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:20,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => d51b53151407a9b9ad82a60ad302f998, NAME => 'TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:20,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d51b53151407a9b9ad82a60ad302f998, NAME => 'TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d51b53151407a9b9ad82a60ad302f998, NAME => 'TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:20,573 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:20,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40778 deadline: 1731278360572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:20,574 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:20,574 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:20,574 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:20,631 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-10T22:38:20,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. 2024-11-10T22:38:20,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. as already flushing 2024-11-10T22:38:20,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. 2024-11-10T22:38:20,632 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => d51b53151407a9b9ad82a60ad302f998, NAME => 'TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:20,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => d51b53151407a9b9ad82a60ad302f998, NAME => 'TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:20,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d51b53151407a9b9ad82a60ad302f998, NAME => 'TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d51b53151407a9b9ad82a60ad302f998, NAME => 'TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:20,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-10T22:38:20,687 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:20,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40738 deadline: 1731278360683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:20,687 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:20,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40736 deadline: 1731278360683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:20,687 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 849d0d71310f,43753,1731278250663 for 'test_row_0' in d51b53151407a9b9ad82a60ad302f998 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2153 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-10T22:38:20,688 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:20,688 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:20,688 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:20,688 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 849d0d71310f,43753,1731278250663 for 'test_row_2' in d51b53151407a9b9ad82a60ad302f998 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2153 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-10T22:38:20,688 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:20,688 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:20,688 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:20,692 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:20,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40754 deadline: 1731278360692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:20,693 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:20,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40774 deadline: 1731278360692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:20,693 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 849d0d71310f,43753,1731278250663 for 'test_row_2' in d51b53151407a9b9ad82a60ad302f998 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2160 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-10T22:38:20,693 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:20,693 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:20,693 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:20,693 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 849d0d71310f,43753,1731278250663 for 'test_row_2' in d51b53151407a9b9ad82a60ad302f998 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2157 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-10T22:38:20,694 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:20,694 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:20,694 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:20,705 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=281, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/A/788648d6317e475d9691263443b8e150 2024-11-10T22:38:20,714 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/B/d8609fed597b41d4bd27515f92539596 is 50, key is test_row_0/B:col10/1731278299875/Put/seqid=0 2024-11-10T22:38:20,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742040_1216 (size=12301) 2024-11-10T22:38:20,785 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-10T22:38:20,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. 2024-11-10T22:38:20,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. as already flushing 2024-11-10T22:38:20,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. 2024-11-10T22:38:20,785 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => d51b53151407a9b9ad82a60ad302f998, NAME => 'TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:20,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => d51b53151407a9b9ad82a60ad302f998, NAME => 'TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:20,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d51b53151407a9b9ad82a60ad302f998, NAME => 'TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d51b53151407a9b9ad82a60ad302f998, NAME => 'TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:20,938 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-10T22:38:20,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. 2024-11-10T22:38:20,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. as already flushing 2024-11-10T22:38:20,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. 2024-11-10T22:38:20,939 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => d51b53151407a9b9ad82a60ad302f998, NAME => 'TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:20,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => d51b53151407a9b9ad82a60ad302f998, NAME => 'TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:20,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d51b53151407a9b9ad82a60ad302f998, NAME => 'TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d51b53151407a9b9ad82a60ad302f998, NAME => 'TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:20,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-10T22:38:21,087 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:21,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40778 deadline: 1731278361083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:21,087 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:21,088 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:21,088 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:21,092 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-10T22:38:21,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. 2024-11-10T22:38:21,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. as already flushing 2024-11-10T22:38:21,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. 2024-11-10T22:38:21,092 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => d51b53151407a9b9ad82a60ad302f998, NAME => 'TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:21,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => d51b53151407a9b9ad82a60ad302f998, NAME => 'TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:21,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d51b53151407a9b9ad82a60ad302f998, NAME => 'TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d51b53151407a9b9ad82a60ad302f998, NAME => 'TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:21,119 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=281 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/B/d8609fed597b41d4bd27515f92539596 2024-11-10T22:38:21,128 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/C/cdc99b7a1f794e2899c42d096d5fc1a8 is 50, key is test_row_0/C:col10/1731278299875/Put/seqid=0 2024-11-10T22:38:21,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742041_1217 (size=12301) 2024-11-10T22:38:21,140 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=281 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/C/cdc99b7a1f794e2899c42d096d5fc1a8 2024-11-10T22:38:21,146 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/A/788648d6317e475d9691263443b8e150 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/788648d6317e475d9691263443b8e150 2024-11-10T22:38:21,153 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/788648d6317e475d9691263443b8e150, entries=200, sequenceid=281, filesize=39.0 K 2024-11-10T22:38:21,154 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/B/d8609fed597b41d4bd27515f92539596 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B/d8609fed597b41d4bd27515f92539596 2024-11-10T22:38:21,161 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B/d8609fed597b41d4bd27515f92539596, entries=150, sequenceid=281, filesize=12.0 K 2024-11-10T22:38:21,162 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/C/cdc99b7a1f794e2899c42d096d5fc1a8 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/C/cdc99b7a1f794e2899c42d096d5fc1a8 2024-11-10T22:38:21,167 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/C/cdc99b7a1f794e2899c42d096d5fc1a8, entries=150, sequenceid=281, filesize=12.0 K 2024-11-10T22:38:21,168 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for d51b53151407a9b9ad82a60ad302f998 in 1292ms, sequenceid=281, compaction requested=false 2024-11-10T22:38:21,168 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d51b53151407a9b9ad82a60ad302f998: 2024-11-10T22:38:21,245 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-10T22:38:21,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. 2024-11-10T22:38:21,246 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2902): Flushing d51b53151407a9b9ad82a60ad302f998 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-10T22:38:21,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411100e88f8e80f8143f2a577374cd86795c2_d51b53151407a9b9ad82a60ad302f998 is 50, key is test_row_0/A:col10/1731278299943/Put/seqid=0 2024-11-10T22:38:21,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742042_1218 (size=12454) 2024-11-10T22:38:21,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-10T22:38:21,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:21,666 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411100e88f8e80f8143f2a577374cd86795c2_d51b53151407a9b9ad82a60ad302f998 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411100e88f8e80f8143f2a577374cd86795c2_d51b53151407a9b9ad82a60ad302f998 2024-11-10T22:38:21,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/A/adc130dbc8e448bd8bce9f694c7e9860, store: [table=TestAcidGuarantees family=A region=d51b53151407a9b9ad82a60ad302f998] 2024-11-10T22:38:21,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/A/adc130dbc8e448bd8bce9f694c7e9860 is 175, key is test_row_0/A:col10/1731278299943/Put/seqid=0 2024-11-10T22:38:21,672 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742043_1219 (size=31255) 2024-11-10T22:38:21,676 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=301, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/A/adc130dbc8e448bd8bce9f694c7e9860 2024-11-10T22:38:21,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/B/e34ba7aebece45968f384f5a064604fc is 50, key is test_row_0/B:col10/1731278299943/Put/seqid=0 2024-11-10T22:38:21,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742044_1220 (size=12301) 2024-11-10T22:38:21,688 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=301 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/B/e34ba7aebece45968f384f5a064604fc 2024-11-10T22:38:21,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/C/3321e90762b94aaca2c0818a8ebb56c0 is 50, key is test_row_0/C:col10/1731278299943/Put/seqid=0 2024-11-10T22:38:21,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742045_1221 (size=12301) 2024-11-10T22:38:22,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(8855): Flush requested on d51b53151407a9b9ad82a60ad302f998 2024-11-10T22:38:22,106 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. as already flushing 2024-11-10T22:38:22,109 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=301 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/C/3321e90762b94aaca2c0818a8ebb56c0 2024-11-10T22:38:22,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/A/adc130dbc8e448bd8bce9f694c7e9860 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/adc130dbc8e448bd8bce9f694c7e9860 2024-11-10T22:38:22,119 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/adc130dbc8e448bd8bce9f694c7e9860, entries=150, sequenceid=301, filesize=30.5 K 2024-11-10T22:38:22,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/B/e34ba7aebece45968f384f5a064604fc as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B/e34ba7aebece45968f384f5a064604fc 2024-11-10T22:38:22,126 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B/e34ba7aebece45968f384f5a064604fc, entries=150, sequenceid=301, filesize=12.0 K 2024-11-10T22:38:22,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/C/3321e90762b94aaca2c0818a8ebb56c0 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/C/3321e90762b94aaca2c0818a8ebb56c0 2024-11-10T22:38:22,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:22,132 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/C/3321e90762b94aaca2c0818a8ebb56c0, entries=150, sequenceid=301, filesize=12.0 K 2024-11-10T22:38:22,133 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=46.96 KB/48090 for d51b53151407a9b9ad82a60ad302f998 in 887ms, sequenceid=301, compaction requested=true 2024-11-10T22:38:22,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for d51b53151407a9b9ad82a60ad302f998: 2024-11-10T22:38:22,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. 2024-11-10T22:38:22,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-11-10T22:38:22,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:22,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-11-10T22:38:22,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:22,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:22,137 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-11-10T22:38:22,137 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8100 sec 2024-11-10T22:38:22,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:22,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:22,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:22,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:22,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:22,140 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 1.8150 sec 2024-11-10T22:38:22,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:22,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:22,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:22,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:22,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(8855): Flush requested on d51b53151407a9b9ad82a60ad302f998 2024-11-10T22:38:22,142 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T22:38:22,142 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d51b53151407a9b9ad82a60ad302f998 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-10T22:38:22,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:22,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:22,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:22,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:22,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:22,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:22,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:22,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:22,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:22,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:22,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:22,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:22,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:22,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:22,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:22,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:22,150 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111071cff20015bc4929acb6727ad2c81ccb_d51b53151407a9b9ad82a60ad302f998 is 50, key is test_row_0/A:col10/1731278302140/Put/seqid=0 2024-11-10T22:38:22,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:22,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:22,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:22,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:22,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:22,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:22,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:22,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:22,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:22,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:22,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:22,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:22,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:22,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:22,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:22,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:22,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:22,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:22,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:22,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:22,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:22,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:22,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:22,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:22,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:22,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:22,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:22,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:22,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:22,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:22,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:22,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:22,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:22,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:22,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:22,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:22,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:22,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:22,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:22,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742046_1222 (size=17534) 2024-11-10T22:38:22,168 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:22,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:22,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:22,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:22,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:22,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:22,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:22,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:22,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:22,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:22,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:22,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:22,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:22,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:22,174 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111071cff20015bc4929acb6727ad2c81ccb_d51b53151407a9b9ad82a60ad302f998 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111071cff20015bc4929acb6727ad2c81ccb_d51b53151407a9b9ad82a60ad302f998 2024-11-10T22:38:22,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:22,175 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/A/518cff827f854a1d89d1822bc9ad8b51, store: [table=TestAcidGuarantees family=A region=d51b53151407a9b9ad82a60ad302f998] 2024-11-10T22:38:22,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:22,175 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/A/518cff827f854a1d89d1822bc9ad8b51 is 175, key is test_row_0/A:col10/1731278302140/Put/seqid=0 2024-11-10T22:38:22,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:22,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:22,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:22,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:22,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:22,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:22,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:22,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:22,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:22,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:22,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:22,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742047_1223 (size=48635) 2024-11-10T22:38:22,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:22,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:22,183 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=312, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/A/518cff827f854a1d89d1822bc9ad8b51 2024-11-10T22:38:22,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:22,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:22,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:22,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:22,195 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/B/08ce541f4a6840e5bfb9a2d8ff77bdaf is 50, key is test_row_0/B:col10/1731278302140/Put/seqid=0 2024-11-10T22:38:22,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742048_1224 (size=14737) 2024-11-10T22:38:22,201 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=312 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/B/08ce541f4a6840e5bfb9a2d8ff77bdaf 2024-11-10T22:38:22,210 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/C/a1cc561efc054245b49dc4ddaab9b595 is 50, key is test_row_0/C:col10/1731278302140/Put/seqid=0 2024-11-10T22:38:22,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742049_1225 (size=9857) 2024-11-10T22:38:22,288 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:22,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40778 deadline: 1731278362285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:22,289 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:22,289 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:22,289 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:22,397 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:22,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40778 deadline: 1731278362392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:22,398 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:22,398 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:22,398 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:22,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-10T22:38:22,472 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T22:38:22,473 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T22:38:22,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees 2024-11-10T22:38:22,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-10T22:38:22,475 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T22:38:22,475 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T22:38:22,476 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T22:38:22,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-10T22:38:22,603 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:22,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40778 deadline: 1731278362602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:22,603 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:22,603 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:22,603 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:22,619 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=312 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/C/a1cc561efc054245b49dc4ddaab9b595 2024-11-10T22:38:22,624 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/A/518cff827f854a1d89d1822bc9ad8b51 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/518cff827f854a1d89d1822bc9ad8b51 2024-11-10T22:38:22,628 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-10T22:38:22,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. 2024-11-10T22:38:22,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. as already flushing 2024-11-10T22:38:22,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. 2024-11-10T22:38:22,628 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => d51b53151407a9b9ad82a60ad302f998, NAME => 'TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:22,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => d51b53151407a9b9ad82a60ad302f998, NAME => 'TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:22,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d51b53151407a9b9ad82a60ad302f998, NAME => 'TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d51b53151407a9b9ad82a60ad302f998, NAME => 'TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:22,633 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/518cff827f854a1d89d1822bc9ad8b51, entries=250, sequenceid=312, filesize=47.5 K 2024-11-10T22:38:22,635 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/B/08ce541f4a6840e5bfb9a2d8ff77bdaf as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B/08ce541f4a6840e5bfb9a2d8ff77bdaf 2024-11-10T22:38:22,640 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B/08ce541f4a6840e5bfb9a2d8ff77bdaf, entries=200, sequenceid=312, filesize=14.4 K 2024-11-10T22:38:22,641 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/C/a1cc561efc054245b49dc4ddaab9b595 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/C/a1cc561efc054245b49dc4ddaab9b595 2024-11-10T22:38:22,646 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/C/a1cc561efc054245b49dc4ddaab9b595, entries=100, sequenceid=312, filesize=9.6 K 2024-11-10T22:38:22,647 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for d51b53151407a9b9ad82a60ad302f998 in 505ms, sequenceid=312, compaction requested=true 2024-11-10T22:38:22,647 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d51b53151407a9b9ad82a60ad302f998: 2024-11-10T22:38:22,647 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d51b53151407a9b9ad82a60ad302f998:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T22:38:22,647 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T22:38:22,647 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d51b53151407a9b9ad82a60ad302f998:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T22:38:22,647 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T22:38:22,647 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T22:38:22,648 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d51b53151407a9b9ad82a60ad302f998:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T22:38:22,648 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T22:38:22,648 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T22:38:22,649 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 151658 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T22:38:22,649 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52204 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T22:38:22,649 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HStore(1541): d51b53151407a9b9ad82a60ad302f998/A is initiating minor compaction (all files) 2024-11-10T22:38:22,649 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1541): d51b53151407a9b9ad82a60ad302f998/B is initiating minor compaction (all files) 2024-11-10T22:38:22,649 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d51b53151407a9b9ad82a60ad302f998/A in TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. 2024-11-10T22:38:22,649 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d51b53151407a9b9ad82a60ad302f998/B in TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. 2024-11-10T22:38:22,649 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B/6176c8a97b074e45aeb42701dd1e9ec4, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B/d8609fed597b41d4bd27515f92539596, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B/e34ba7aebece45968f384f5a064604fc, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B/08ce541f4a6840e5bfb9a2d8ff77bdaf] into tmpdir=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp, totalSize=51.0 K 2024-11-10T22:38:22,649 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/8c0efc7b662e40fb8f682f6cb9fcd6ff, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/788648d6317e475d9691263443b8e150, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/adc130dbc8e448bd8bce9f694c7e9860, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/518cff827f854a1d89d1822bc9ad8b51] into tmpdir=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp, totalSize=148.1 K 2024-11-10T22:38:22,649 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. 2024-11-10T22:38:22,650 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. files: [hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/8c0efc7b662e40fb8f682f6cb9fcd6ff, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/788648d6317e475d9691263443b8e150, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/adc130dbc8e448bd8bce9f694c7e9860, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/518cff827f854a1d89d1822bc9ad8b51] 2024-11-10T22:38:22,650 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting 6176c8a97b074e45aeb42701dd1e9ec4, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1731278297839 2024-11-10T22:38:22,650 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8c0efc7b662e40fb8f682f6cb9fcd6ff, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1731278297839 2024-11-10T22:38:22,650 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting d8609fed597b41d4bd27515f92539596, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1731278298532 2024-11-10T22:38:22,651 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 788648d6317e475d9691263443b8e150, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1731278298531 2024-11-10T22:38:22,651 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting e34ba7aebece45968f384f5a064604fc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1731278299924 2024-11-10T22:38:22,651 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting adc130dbc8e448bd8bce9f694c7e9860, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1731278299924 2024-11-10T22:38:22,651 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting 08ce541f4a6840e5bfb9a2d8ff77bdaf, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1731278302117 2024-11-10T22:38:22,652 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 518cff827f854a1d89d1822bc9ad8b51, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1731278302111 2024-11-10T22:38:22,666 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d51b53151407a9b9ad82a60ad302f998#B#compaction#188 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T22:38:22,667 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/B/117a94cb724d40808ca895e4e3d80ffa is 50, key is test_row_0/B:col10/1731278302140/Put/seqid=0 2024-11-10T22:38:22,668 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d51b53151407a9b9ad82a60ad302f998] 2024-11-10T22:38:22,679 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411108a66034fed7d4098a9d89b7e3cc3620a_d51b53151407a9b9ad82a60ad302f998 store=[table=TestAcidGuarantees family=A region=d51b53151407a9b9ad82a60ad302f998] 2024-11-10T22:38:22,681 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411108a66034fed7d4098a9d89b7e3cc3620a_d51b53151407a9b9ad82a60ad302f998, store=[table=TestAcidGuarantees family=A region=d51b53151407a9b9ad82a60ad302f998] 2024-11-10T22:38:22,682 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411108a66034fed7d4098a9d89b7e3cc3620a_d51b53151407a9b9ad82a60ad302f998 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d51b53151407a9b9ad82a60ad302f998] 2024-11-10T22:38:22,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742050_1226 (size=13051) 2024-11-10T22:38:22,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742051_1227 (size=4469) 2024-11-10T22:38:22,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(8855): Flush requested on d51b53151407a9b9ad82a60ad302f998 2024-11-10T22:38:22,707 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T22:38:22,708 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d51b53151407a9b9ad82a60ad302f998 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-10T22:38:22,717 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111083fe874360064eb79e3c7740f4cf200b_d51b53151407a9b9ad82a60ad302f998 is 50, key is test_row_0/A:col10/1731278302274/Put/seqid=0 2024-11-10T22:38:22,724 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:22,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40736 deadline: 1731278362721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:22,725 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:22,725 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:22,725 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:22,725 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:22,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40774 deadline: 1731278362723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:22,725 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:22,725 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:22,725 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:22,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742052_1228 (size=14994) 2024-11-10T22:38:22,727 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:22,731 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:22,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40738 deadline: 1731278362724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:22,731 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:22,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40754 deadline: 1731278362725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:22,732 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111083fe874360064eb79e3c7740f4cf200b_d51b53151407a9b9ad82a60ad302f998 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111083fe874360064eb79e3c7740f4cf200b_d51b53151407a9b9ad82a60ad302f998 2024-11-10T22:38:22,732 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:22,732 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:22,732 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:22,733 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:22,733 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:22,733 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:22,734 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/A/03ee732a8ba04a35a098f9c3d835945b, store: [table=TestAcidGuarantees family=A region=d51b53151407a9b9ad82a60ad302f998] 2024-11-10T22:38:22,734 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/A/03ee732a8ba04a35a098f9c3d835945b is 175, key is test_row_0/A:col10/1731278302274/Put/seqid=0 2024-11-10T22:38:22,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742053_1229 (size=39949) 2024-11-10T22:38:22,740 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=338, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/A/03ee732a8ba04a35a098f9c3d835945b 2024-11-10T22:38:22,752 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/B/b67e9ac5e9fd4995a8ad930ee9082032 is 50, key is test_row_0/B:col10/1731278302274/Put/seqid=0 2024-11-10T22:38:22,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742054_1230 (size=12301) 2024-11-10T22:38:22,781 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-10T22:38:22,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. 2024-11-10T22:38:22,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. as already flushing 2024-11-10T22:38:22,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. 2024-11-10T22:38:22,781 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => d51b53151407a9b9ad82a60ad302f998, NAME => 'TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:22,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => d51b53151407a9b9ad82a60ad302f998, NAME => 'TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:22,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d51b53151407a9b9ad82a60ad302f998, NAME => 'TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d51b53151407a9b9ad82a60ad302f998, NAME => 'TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:22,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-10T22:38:22,836 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:22,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40736 deadline: 1731278362833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:22,837 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:22,837 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:22,837 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:22,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40774 deadline: 1731278362833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:22,837 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:22,837 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:22,837 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:22,837 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:22,843 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:22,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40754 deadline: 1731278362842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:22,843 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:22,843 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:22,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40738 deadline: 1731278362842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:22,843 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:22,843 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:22,844 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:22,844 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:22,844 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:22,920 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:22,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40778 deadline: 1731278362913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:22,920 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:22,921 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:22,921 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:22,935 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-10T22:38:22,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. 2024-11-10T22:38:22,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. as already flushing 2024-11-10T22:38:22,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. 2024-11-10T22:38:22,935 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => d51b53151407a9b9ad82a60ad302f998, NAME => 'TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:22,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => d51b53151407a9b9ad82a60ad302f998, NAME => 'TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:22,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d51b53151407a9b9ad82a60ad302f998, NAME => 'TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d51b53151407a9b9ad82a60ad302f998, NAME => 'TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:23,047 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:23,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40736 deadline: 1731278363042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:23,048 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:23,048 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:23,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40774 deadline: 1731278363042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:23,048 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:23,048 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:23,048 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:23,048 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:23,048 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:23,053 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:23,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40738 deadline: 1731278363052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:23,053 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:23,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40754 deadline: 1731278363052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:23,053 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:23,053 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:23,053 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:23,053 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:23,053 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:23,053 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:23,088 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/B/117a94cb724d40808ca895e4e3d80ffa as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B/117a94cb724d40808ca895e4e3d80ffa 2024-11-10T22:38:23,088 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-10T22:38:23,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. 2024-11-10T22:38:23,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. as already flushing 2024-11-10T22:38:23,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. 2024-11-10T22:38:23,089 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => d51b53151407a9b9ad82a60ad302f998, NAME => 'TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:23,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => d51b53151407a9b9ad82a60ad302f998, NAME => 'TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:23,089 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d51b53151407a9b9ad82a60ad302f998#A#compaction#189 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T22:38:23,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d51b53151407a9b9ad82a60ad302f998, NAME => 'TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d51b53151407a9b9ad82a60ad302f998, NAME => 'TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:23,090 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/A/f9c3d2b4688e4cfd8ec2dc98bdadd648 is 175, key is test_row_0/A:col10/1731278302140/Put/seqid=0 2024-11-10T22:38:23,099 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d51b53151407a9b9ad82a60ad302f998/B of d51b53151407a9b9ad82a60ad302f998 into 117a94cb724d40808ca895e4e3d80ffa(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T22:38:23,099 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d51b53151407a9b9ad82a60ad302f998: 2024-11-10T22:38:23,099 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., storeName=d51b53151407a9b9ad82a60ad302f998/B, priority=12, startTime=1731278302647; duration=0sec 2024-11-10T22:38:23,099 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T22:38:23,099 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d51b53151407a9b9ad82a60ad302f998:B 2024-11-10T22:38:23,099 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T22:38:23,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-10T22:38:23,103 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47290 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T22:38:23,103 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1541): d51b53151407a9b9ad82a60ad302f998/C is initiating minor compaction (all files) 2024-11-10T22:38:23,103 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d51b53151407a9b9ad82a60ad302f998/C in TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. 2024-11-10T22:38:23,103 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/C/63304d55ae4e4fafaa483faac1592978, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/C/cdc99b7a1f794e2899c42d096d5fc1a8, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/C/3321e90762b94aaca2c0818a8ebb56c0, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/C/a1cc561efc054245b49dc4ddaab9b595] into tmpdir=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp, totalSize=46.2 K 2024-11-10T22:38:23,104 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting 63304d55ae4e4fafaa483faac1592978, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1731278297839 2024-11-10T22:38:23,104 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting cdc99b7a1f794e2899c42d096d5fc1a8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1731278298532 2024-11-10T22:38:23,104 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting 3321e90762b94aaca2c0818a8ebb56c0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1731278299924 2024-11-10T22:38:23,105 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting a1cc561efc054245b49dc4ddaab9b595, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1731278302117 2024-11-10T22:38:23,121 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d51b53151407a9b9ad82a60ad302f998#C#compaction#192 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T22:38:23,121 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/C/5d1244c0024545a0a467e8ea1be52b3d is 50, key is test_row_0/C:col10/1731278302140/Put/seqid=0 2024-11-10T22:38:23,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742055_1231 (size=32112) 2024-11-10T22:38:23,130 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/A/f9c3d2b4688e4cfd8ec2dc98bdadd648 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/f9c3d2b4688e4cfd8ec2dc98bdadd648 2024-11-10T22:38:23,136 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d51b53151407a9b9ad82a60ad302f998/A of d51b53151407a9b9ad82a60ad302f998 into f9c3d2b4688e4cfd8ec2dc98bdadd648(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T22:38:23,136 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d51b53151407a9b9ad82a60ad302f998: 2024-11-10T22:38:23,137 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., storeName=d51b53151407a9b9ad82a60ad302f998/A, priority=12, startTime=1731278302647; duration=0sec 2024-11-10T22:38:23,137 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T22:38:23,137 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d51b53151407a9b9ad82a60ad302f998:A 2024-11-10T22:38:23,140 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742056_1232 (size=13017) 2024-11-10T22:38:23,147 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/C/5d1244c0024545a0a467e8ea1be52b3d as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/C/5d1244c0024545a0a467e8ea1be52b3d 2024-11-10T22:38:23,155 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d51b53151407a9b9ad82a60ad302f998/C of d51b53151407a9b9ad82a60ad302f998 into 5d1244c0024545a0a467e8ea1be52b3d(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T22:38:23,155 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d51b53151407a9b9ad82a60ad302f998: 2024-11-10T22:38:23,155 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., storeName=d51b53151407a9b9ad82a60ad302f998/C, priority=12, startTime=1731278302647; duration=0sec 2024-11-10T22:38:23,156 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T22:38:23,156 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d51b53151407a9b9ad82a60ad302f998:C 2024-11-10T22:38:23,163 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=338 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/B/b67e9ac5e9fd4995a8ad930ee9082032 2024-11-10T22:38:23,176 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/C/7cd7c715da9944fd9624383ceeac77c0 is 50, key is test_row_0/C:col10/1731278302274/Put/seqid=0 2024-11-10T22:38:23,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742057_1233 (size=12301) 2024-11-10T22:38:23,241 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-10T22:38:23,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. 2024-11-10T22:38:23,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. as already flushing 2024-11-10T22:38:23,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. 2024-11-10T22:38:23,242 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => d51b53151407a9b9ad82a60ad302f998, NAME => 'TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:23,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => d51b53151407a9b9ad82a60ad302f998, NAME => 'TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:23,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d51b53151407a9b9ad82a60ad302f998, NAME => 'TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d51b53151407a9b9ad82a60ad302f998, NAME => 'TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:23,354 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:23,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40736 deadline: 1731278363352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:23,355 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:23,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40774 deadline: 1731278363352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:23,355 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:23,355 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:23,355 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:23,355 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:23,355 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:23,355 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:23,362 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:23,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40738 deadline: 1731278363362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:23,363 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:23,363 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:23,363 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:23,364 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:23,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40754 deadline: 1731278363362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:23,365 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:23,365 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:23,365 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:23,395 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-10T22:38:23,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. 2024-11-10T22:38:23,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. as already flushing 2024-11-10T22:38:23,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. 2024-11-10T22:38:23,395 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => d51b53151407a9b9ad82a60ad302f998, NAME => 'TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:23,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => d51b53151407a9b9ad82a60ad302f998, NAME => 'TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:23,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d51b53151407a9b9ad82a60ad302f998, NAME => 'TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d51b53151407a9b9ad82a60ad302f998, NAME => 'TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:23,437 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:23,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40778 deadline: 1731278363432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:23,437 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:23,437 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:23,437 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:23,548 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-10T22:38:23,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. 2024-11-10T22:38:23,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. as already flushing 2024-11-10T22:38:23,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. 2024-11-10T22:38:23,548 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => d51b53151407a9b9ad82a60ad302f998, NAME => 'TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:23,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => d51b53151407a9b9ad82a60ad302f998, NAME => 'TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:23,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d51b53151407a9b9ad82a60ad302f998, NAME => 'TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d51b53151407a9b9ad82a60ad302f998, NAME => 'TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:23,581 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=338 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/C/7cd7c715da9944fd9624383ceeac77c0 2024-11-10T22:38:23,586 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/A/03ee732a8ba04a35a098f9c3d835945b as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/03ee732a8ba04a35a098f9c3d835945b 2024-11-10T22:38:23,590 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/03ee732a8ba04a35a098f9c3d835945b, entries=200, sequenceid=338, filesize=39.0 K 2024-11-10T22:38:23,591 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/B/b67e9ac5e9fd4995a8ad930ee9082032 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B/b67e9ac5e9fd4995a8ad930ee9082032 2024-11-10T22:38:23,596 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B/b67e9ac5e9fd4995a8ad930ee9082032, entries=150, sequenceid=338, filesize=12.0 K 2024-11-10T22:38:23,597 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/C/7cd7c715da9944fd9624383ceeac77c0 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/C/7cd7c715da9944fd9624383ceeac77c0 2024-11-10T22:38:23,601 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/C/7cd7c715da9944fd9624383ceeac77c0, entries=150, sequenceid=338, filesize=12.0 K 2024-11-10T22:38:23,602 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=53.67 KB/54960 for d51b53151407a9b9ad82a60ad302f998 in 895ms, sequenceid=338, compaction requested=false 2024-11-10T22:38:23,602 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d51b53151407a9b9ad82a60ad302f998: 2024-11-10T22:38:23,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-10T22:38:23,701 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-10T22:38:23,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. 2024-11-10T22:38:23,702 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2902): Flushing d51b53151407a9b9ad82a60ad302f998 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-10T22:38:23,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110c9107bf7237c4b3288d6c7bde8b9b8c3_d51b53151407a9b9ad82a60ad302f998 is 50, key is test_row_0/A:col10/1731278302721/Put/seqid=0 2024-11-10T22:38:23,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742058_1234 (size=12454) 2024-11-10T22:38:23,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(8855): Flush requested on d51b53151407a9b9ad82a60ad302f998 2024-11-10T22:38:23,867 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. as already flushing 2024-11-10T22:38:23,925 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:23,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40774 deadline: 1731278363917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:23,926 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:23,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40738 deadline: 1731278363917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:23,926 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:23,926 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:23,926 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:23,926 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:23,927 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:23,927 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:23,928 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:23,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40736 deadline: 1731278363921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:23,929 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:23,929 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:23,929 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:23,929 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:23,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40754 deadline: 1731278363922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:23,929 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:23,929 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:23,929 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:24,035 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:24,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40774 deadline: 1731278364032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:24,035 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:24,036 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:24,036 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:24,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40736 deadline: 1731278364033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:24,036 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:24,036 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:24,036 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:24,036 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:24,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40754 deadline: 1731278364033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:24,036 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:24,036 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:24,036 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:24,036 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:24,036 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:24,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40738 deadline: 1731278364033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:24,037 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:24,037 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:24,037 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:24,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:24,121 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110c9107bf7237c4b3288d6c7bde8b9b8c3_d51b53151407a9b9ad82a60ad302f998 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110c9107bf7237c4b3288d6c7bde8b9b8c3_d51b53151407a9b9ad82a60ad302f998 2024-11-10T22:38:24,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/A/ee8f97c1d0754baa8271a277d811c0a7, store: [table=TestAcidGuarantees family=A region=d51b53151407a9b9ad82a60ad302f998] 2024-11-10T22:38:24,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/A/ee8f97c1d0754baa8271a277d811c0a7 is 175, key is test_row_0/A:col10/1731278302721/Put/seqid=0 2024-11-10T22:38:24,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742059_1235 (size=31255) 2024-11-10T22:38:24,245 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:24,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40774 deadline: 1731278364242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:24,246 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:24,246 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:24,246 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:24,246 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:24,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40736 deadline: 1731278364242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:24,246 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:24,246 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:24,246 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:24,246 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:24,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40754 deadline: 1731278364242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:24,246 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:24,246 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:24,246 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:24,246 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:24,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40738 deadline: 1731278364242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:24,247 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:24,247 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:24,247 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:24,455 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:24,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40778 deadline: 1731278364452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:24,456 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 849d0d71310f,43753,1731278250663 for 'test_row_2' in d51b53151407a9b9ad82a60ad302f998 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2171 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-10T22:38:24,457 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:24,457 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:24,457 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:24,527 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=352, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/A/ee8f97c1d0754baa8271a277d811c0a7 2024-11-10T22:38:24,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/B/21608d1a1e6e484699ee3b75ebbe7da2 is 50, key is test_row_0/B:col10/1731278302721/Put/seqid=0 2024-11-10T22:38:24,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742060_1236 (size=12301) 2024-11-10T22:38:24,556 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:24,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40774 deadline: 1731278364552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:24,557 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:24,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40736 deadline: 1731278364552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:24,557 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:24,557 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:24,557 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:24,557 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:24,557 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:24,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40738 deadline: 1731278364552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:24,557 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:24,557 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:24,557 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:24,557 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:24,557 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:24,557 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:24,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40754 deadline: 1731278364552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:24,558 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:24,558 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:24,558 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:24,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-10T22:38:24,942 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=352 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/B/21608d1a1e6e484699ee3b75ebbe7da2 2024-11-10T22:38:24,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/C/2bbd3474af6347c781c4168cc2cebc6b is 50, key is test_row_0/C:col10/1731278302721/Put/seqid=0 2024-11-10T22:38:24,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742061_1237 (size=12301) 2024-11-10T22:38:25,066 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:25,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40774 deadline: 1731278365062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:25,066 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:25,067 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:25,067 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:25,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40738 deadline: 1731278365063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:25,067 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:25,067 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:25,067 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:25,067 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:25,067 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:25,067 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:25,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40736 deadline: 1731278365063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:25,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40754 deadline: 1731278365063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:25,067 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:25,067 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:25,067 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:25,067 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:25,067 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:25,067 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:25,356 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=352 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/C/2bbd3474af6347c781c4168cc2cebc6b 2024-11-10T22:38:25,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/A/ee8f97c1d0754baa8271a277d811c0a7 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/ee8f97c1d0754baa8271a277d811c0a7 2024-11-10T22:38:25,366 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/ee8f97c1d0754baa8271a277d811c0a7, entries=150, sequenceid=352, filesize=30.5 K 2024-11-10T22:38:25,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/B/21608d1a1e6e484699ee3b75ebbe7da2 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B/21608d1a1e6e484699ee3b75ebbe7da2 2024-11-10T22:38:25,371 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B/21608d1a1e6e484699ee3b75ebbe7da2, entries=150, sequenceid=352, filesize=12.0 K 2024-11-10T22:38:25,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/C/2bbd3474af6347c781c4168cc2cebc6b as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/C/2bbd3474af6347c781c4168cc2cebc6b 2024-11-10T22:38:25,377 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/C/2bbd3474af6347c781c4168cc2cebc6b, entries=150, sequenceid=352, filesize=12.0 K 2024-11-10T22:38:25,378 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for d51b53151407a9b9ad82a60ad302f998 in 1676ms, sequenceid=352, compaction requested=true 2024-11-10T22:38:25,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2603): Flush status journal for d51b53151407a9b9ad82a60ad302f998: 2024-11-10T22:38:25,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. 2024-11-10T22:38:25,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=53 2024-11-10T22:38:25,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.HMaster(4169): Remote procedure done, pid=53 2024-11-10T22:38:25,381 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-11-10T22:38:25,381 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.9030 sec 2024-11-10T22:38:25,383 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees in 2.9090 sec 2024-11-10T22:38:26,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(8855): Flush requested on d51b53151407a9b9ad82a60ad302f998 2024-11-10T22:38:26,077 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T22:38:26,077 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d51b53151407a9b9ad82a60ad302f998 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-10T22:38:26,084 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111064cb5a25a5624021815db288a2c83b65_d51b53151407a9b9ad82a60ad302f998 is 50, key is test_row_0/A:col10/1731278303920/Put/seqid=0 2024-11-10T22:38:26,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742062_1238 (size=17534) 2024-11-10T22:38:26,089 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:26,090 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:26,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40754 deadline: 1731278366086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:26,091 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:26,091 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:26,091 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:26,091 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:26,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40736 deadline: 1731278366091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:26,092 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:26,092 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:26,092 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:26,092 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:26,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40738 deadline: 1731278366091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:26,092 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:26,092 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:26,092 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:26,094 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:26,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40774 deadline: 1731278366091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:26,094 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:26,094 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:26,094 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:26,094 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111064cb5a25a5624021815db288a2c83b65_d51b53151407a9b9ad82a60ad302f998 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111064cb5a25a5624021815db288a2c83b65_d51b53151407a9b9ad82a60ad302f998 2024-11-10T22:38:26,095 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/A/5f1dac635c454fe0bee47e308dfed044, store: [table=TestAcidGuarantees family=A region=d51b53151407a9b9ad82a60ad302f998] 2024-11-10T22:38:26,096 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/A/5f1dac635c454fe0bee47e308dfed044 is 175, key is test_row_0/A:col10/1731278303920/Put/seqid=0 2024-11-10T22:38:26,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742063_1239 (size=48639) 2024-11-10T22:38:26,100 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=379, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/A/5f1dac635c454fe0bee47e308dfed044 2024-11-10T22:38:26,110 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/B/95e1dcb92203407f8d6728e0a413675a is 50, key is test_row_0/B:col10/1731278303920/Put/seqid=0 2024-11-10T22:38:26,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742064_1240 (size=12301) 2024-11-10T22:38:26,205 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:26,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40754 deadline: 1731278366203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:26,205 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:26,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40736 deadline: 1731278366203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:26,206 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:26,206 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:26,206 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:26,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40738 deadline: 1731278366203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:26,206 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:26,206 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:26,206 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:26,206 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:26,206 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:26,206 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:26,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40774 deadline: 1731278366203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:26,206 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:26,206 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:26,206 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:26,206 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:26,206 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:26,418 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:26,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40754 deadline: 1731278366413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:26,418 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:26,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40736 deadline: 1731278366413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:26,418 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:26,418 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:26,418 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:26,418 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:26,418 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:26,419 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:26,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40738 deadline: 1731278366413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:26,419 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:26,419 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:26,419 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:26,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40774 deadline: 1731278366413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:26,419 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:26,419 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:26,419 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:26,419 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:26,419 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:26,475 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:26,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40778 deadline: 1731278366472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:26,475 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 849d0d71310f,43753,1731278250663 for 'test_row_2' in d51b53151407a9b9ad82a60ad302f998 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4190 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-10T22:38:26,476 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:26,476 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:26,476 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:26,515 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=379 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/B/95e1dcb92203407f8d6728e0a413675a 2024-11-10T22:38:26,524 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/C/b340498e8adf488aa11fb0b5f0f0780c is 50, key is test_row_0/C:col10/1731278303920/Put/seqid=0 2024-11-10T22:38:26,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742065_1241 (size=12301) 2024-11-10T22:38:26,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-10T22:38:26,643 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T22:38:26,643 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T22:38:26,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees 2024-11-10T22:38:26,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-10T22:38:26,645 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T22:38:26,646 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T22:38:26,646 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T22:38:26,725 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:26,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40754 deadline: 1731278366722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:26,725 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:26,725 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:26,725 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:26,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40736 deadline: 1731278366722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:26,725 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:26,726 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:26,726 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:26,726 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:26,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40738 deadline: 1731278366722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:26,726 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:26,726 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:26,726 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:26,726 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:26,726 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:26,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40774 deadline: 1731278366723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:26,726 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:26,726 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:26,726 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:26,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-10T22:38:26,799 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-10T22:38:26,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. 2024-11-10T22:38:26,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. as already flushing 2024-11-10T22:38:26,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. 2024-11-10T22:38:26,799 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => d51b53151407a9b9ad82a60ad302f998, NAME => 'TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:26,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => d51b53151407a9b9ad82a60ad302f998, NAME => 'TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:26,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d51b53151407a9b9ad82a60ad302f998, NAME => 'TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d51b53151407a9b9ad82a60ad302f998, NAME => 'TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:26,930 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=379 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/C/b340498e8adf488aa11fb0b5f0f0780c 2024-11-10T22:38:26,935 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/A/5f1dac635c454fe0bee47e308dfed044 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/5f1dac635c454fe0bee47e308dfed044 2024-11-10T22:38:26,939 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/5f1dac635c454fe0bee47e308dfed044, entries=250, sequenceid=379, filesize=47.5 K 2024-11-10T22:38:26,940 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/B/95e1dcb92203407f8d6728e0a413675a as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B/95e1dcb92203407f8d6728e0a413675a 2024-11-10T22:38:26,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:26,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:26,944 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B/95e1dcb92203407f8d6728e0a413675a, entries=150, sequenceid=379, filesize=12.0 K 2024-11-10T22:38:26,945 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/C/b340498e8adf488aa11fb0b5f0f0780c as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/C/b340498e8adf488aa11fb0b5f0f0780c 2024-11-10T22:38:26,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:26,949 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/C/b340498e8adf488aa11fb0b5f0f0780c, entries=150, sequenceid=379, filesize=12.0 K 2024-11-10T22:38:26,950 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=46.96 KB/48090 for d51b53151407a9b9ad82a60ad302f998 in 873ms, sequenceid=379, compaction requested=true 2024-11-10T22:38:26,950 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d51b53151407a9b9ad82a60ad302f998: 2024-11-10T22:38:26,951 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d51b53151407a9b9ad82a60ad302f998:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T22:38:26,951 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T22:38:26,951 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T22:38:26,951 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d51b53151407a9b9ad82a60ad302f998:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T22:38:26,951 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T22:38:26,951 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d51b53151407a9b9ad82a60ad302f998:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T22:38:26,951 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T22:38:26,951 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T22:38:26,952 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-10T22:38:26,952 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 151955 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T22:38:26,952 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49954 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T22:38:26,952 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HStore(1541): d51b53151407a9b9ad82a60ad302f998/A is initiating minor compaction (all files) 2024-11-10T22:38:26,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. 2024-11-10T22:38:26,952 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1541): d51b53151407a9b9ad82a60ad302f998/B is initiating minor compaction (all files) 2024-11-10T22:38:26,952 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d51b53151407a9b9ad82a60ad302f998/A in TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. 2024-11-10T22:38:26,952 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d51b53151407a9b9ad82a60ad302f998/B in TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. 2024-11-10T22:38:26,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:26,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:26,952 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/f9c3d2b4688e4cfd8ec2dc98bdadd648, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/03ee732a8ba04a35a098f9c3d835945b, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/ee8f97c1d0754baa8271a277d811c0a7, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/5f1dac635c454fe0bee47e308dfed044] into tmpdir=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp, totalSize=148.4 K 2024-11-10T22:38:26,952 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B/117a94cb724d40808ca895e4e3d80ffa, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B/b67e9ac5e9fd4995a8ad930ee9082032, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B/21608d1a1e6e484699ee3b75ebbe7da2, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B/95e1dcb92203407f8d6728e0a413675a] into tmpdir=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp, totalSize=48.8 K 2024-11-10T22:38:26,952 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. 2024-11-10T22:38:26,952 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. files: [hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/f9c3d2b4688e4cfd8ec2dc98bdadd648, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/03ee732a8ba04a35a098f9c3d835945b, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/ee8f97c1d0754baa8271a277d811c0a7, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/5f1dac635c454fe0bee47e308dfed044] 2024-11-10T22:38:26,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:26,953 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2902): Flushing d51b53151407a9b9ad82a60ad302f998 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-10T22:38:26,953 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting f9c3d2b4688e4cfd8ec2dc98bdadd648, keycount=150, bloomtype=ROW, size=31.4 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1731278299938 2024-11-10T22:38:26,953 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting 117a94cb724d40808ca895e4e3d80ffa, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1731278299938 2024-11-10T22:38:26,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:26,954 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 03ee732a8ba04a35a098f9c3d835945b, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=338, earliestPutTs=1731278302247 2024-11-10T22:38:26,954 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting b67e9ac5e9fd4995a8ad930ee9082032, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=338, earliestPutTs=1731278302274 2024-11-10T22:38:26,954 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting ee8f97c1d0754baa8271a277d811c0a7, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1731278302721 2024-11-10T22:38:26,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:26,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:26,955 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting 21608d1a1e6e484699ee3b75ebbe7da2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1731278302721 2024-11-10T22:38:26,955 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5f1dac635c454fe0bee47e308dfed044, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=379, earliestPutTs=1731278303908 2024-11-10T22:38:26,955 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting 95e1dcb92203407f8d6728e0a413675a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=379, earliestPutTs=1731278303920 2024-11-10T22:38:26,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:26,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:26,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:26,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:26,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:26,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:26,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:26,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:26,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:26,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:26,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:26,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:26,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:26,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:26,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:26,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-10T22:38:26,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:26,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:26,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:26,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:26,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:26,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:26,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:26,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:26,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:26,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:26,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:26,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:26,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:26,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:26,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:26,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:26,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:26,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:26,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:26,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:26,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:26,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:26,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:26,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:26,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:26,974 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d51b53151407a9b9ad82a60ad302f998] 2024-11-10T22:38:26,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:26,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:26,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:26,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:26,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:26,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:26,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:26,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:26,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:26,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:26,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:26,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:26,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:26,982 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d51b53151407a9b9ad82a60ad302f998#B#compaction#200 average throughput is 0.82 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T22:38:26,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:26,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:26,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:26,983 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/B/675f97d30adc4ae7827e9deed06ed49b is 50, key is test_row_0/B:col10/1731278303920/Put/seqid=0 2024-11-10T22:38:26,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:26,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:26,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:26,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:26,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:26,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:26,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:26,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:26,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:26,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:26,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:26,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:26,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:26,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:26,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:26,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:26,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:26,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:26,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:26,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:26,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:26,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:26,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:26,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:26,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:26,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:26,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:26,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:26,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110d7790bd128bf437c90be54014064034b_d51b53151407a9b9ad82a60ad302f998 is 50, key is test_row_0/A:col10/1731278306089/Put/seqid=0 2024-11-10T22:38:26,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:26,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:26,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:26,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:26,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:26,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:26,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:26,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:26,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,000 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111066ee91f9423a452fab6ab51759b5d658_d51b53151407a9b9ad82a60ad302f998 store=[table=TestAcidGuarantees family=A region=d51b53151407a9b9ad82a60ad302f998] 2024-11-10T22:38:27,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,006 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111066ee91f9423a452fab6ab51759b5d658_d51b53151407a9b9ad82a60ad302f998, store=[table=TestAcidGuarantees family=A region=d51b53151407a9b9ad82a60ad302f998] 2024-11-10T22:38:27,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,006 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111066ee91f9423a452fab6ab51759b5d658_d51b53151407a9b9ad82a60ad302f998 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d51b53151407a9b9ad82a60ad302f998] 2024-11-10T22:38:27,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742066_1242 (size=13187) 2024-11-10T22:38:27,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742067_1243 (size=9914) 2024-11-10T22:38:27,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,036 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/B/675f97d30adc4ae7827e9deed06ed49b as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B/675f97d30adc4ae7827e9deed06ed49b 2024-11-10T22:38:27,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,044 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d51b53151407a9b9ad82a60ad302f998/B of d51b53151407a9b9ad82a60ad302f998 into 675f97d30adc4ae7827e9deed06ed49b(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T22:38:27,044 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d51b53151407a9b9ad82a60ad302f998: 2024-11-10T22:38:27,044 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., storeName=d51b53151407a9b9ad82a60ad302f998/B, priority=12, startTime=1731278306951; duration=0sec 2024-11-10T22:38:27,044 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T22:38:27,044 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d51b53151407a9b9ad82a60ad302f998:B 2024-11-10T22:38:27,045 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T22:38:27,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,046 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49920 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T22:38:27,046 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1541): d51b53151407a9b9ad82a60ad302f998/C is initiating minor compaction (all files) 2024-11-10T22:38:27,046 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d51b53151407a9b9ad82a60ad302f998/C in TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. 2024-11-10T22:38:27,046 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/C/5d1244c0024545a0a467e8ea1be52b3d, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/C/7cd7c715da9944fd9624383ceeac77c0, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/C/2bbd3474af6347c781c4168cc2cebc6b, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/C/b340498e8adf488aa11fb0b5f0f0780c] into tmpdir=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp, totalSize=48.8 K 2024-11-10T22:38:27,047 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting 5d1244c0024545a0a467e8ea1be52b3d, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1731278299938 2024-11-10T22:38:27,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,047 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting 7cd7c715da9944fd9624383ceeac77c0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=338, earliestPutTs=1731278302274 2024-11-10T22:38:27,047 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting 2bbd3474af6347c781c4168cc2cebc6b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1731278302721 2024-11-10T22:38:27,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,048 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting b340498e8adf488aa11fb0b5f0f0780c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=379, earliestPutTs=1731278303920 2024-11-10T22:38:27,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742068_1244 (size=4469) 2024-11-10T22:38:27,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,058 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d51b53151407a9b9ad82a60ad302f998#A#compaction#201 average throughput is 0.29 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T22:38:27,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,059 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/A/37a20fb724c34fc09699307bedb23540 is 175, key is test_row_0/A:col10/1731278303920/Put/seqid=0 2024-11-10T22:38:27,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,071 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d51b53151407a9b9ad82a60ad302f998#C#compaction#203 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T22:38:27,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,072 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/C/e4ed274f3413411984807705284474c2 is 50, key is test_row_0/C:col10/1731278303920/Put/seqid=0 2024-11-10T22:38:27,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742069_1245 (size=32141) 2024-11-10T22:38:27,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742070_1246 (size=13153) 2024-11-10T22:38:27,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,111 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/C/e4ed274f3413411984807705284474c2 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/C/e4ed274f3413411984807705284474c2 2024-11-10T22:38:27,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,118 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d51b53151407a9b9ad82a60ad302f998/C of d51b53151407a9b9ad82a60ad302f998 into e4ed274f3413411984807705284474c2(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T22:38:27,118 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d51b53151407a9b9ad82a60ad302f998: 2024-11-10T22:38:27,118 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., storeName=d51b53151407a9b9ad82a60ad302f998/C, priority=12, startTime=1731278306951; duration=0sec 2024-11-10T22:38:27,118 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T22:38:27,118 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d51b53151407a9b9ad82a60ad302f998:C 2024-11-10T22:38:27,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(8855): Flush requested on d51b53151407a9b9ad82a60ad302f998 2024-11-10T22:38:27,263 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. as already flushing 2024-11-10T22:38:27,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-10T22:38:27,307 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:27,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40738 deadline: 1731278367300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:27,307 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:27,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40754 deadline: 1731278367302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:27,308 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:27,308 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:27,308 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:27,308 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:27,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40736 deadline: 1731278367304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:27,308 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:27,308 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:27,309 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:27,309 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:27,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40774 deadline: 1731278367305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:27,309 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:27,309 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:27,309 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:27,309 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:27,309 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:27,309 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:27,415 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:27,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40738 deadline: 1731278367412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:27,415 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:27,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40754 deadline: 1731278367412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:27,416 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:27,416 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:27,416 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:27,416 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:27,416 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:27,416 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:27,416 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:27,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40736 deadline: 1731278367412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:27,416 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:27,416 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:27,416 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:27,416 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:27,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40774 deadline: 1731278367412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:27,417 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:27,417 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:27,417 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:27,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:27,437 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110d7790bd128bf437c90be54014064034b_d51b53151407a9b9ad82a60ad302f998 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110d7790bd128bf437c90be54014064034b_d51b53151407a9b9ad82a60ad302f998 2024-11-10T22:38:27,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/A/70369733dd5a4510b4d3efa4c80fef98, store: [table=TestAcidGuarantees family=A region=d51b53151407a9b9ad82a60ad302f998] 2024-11-10T22:38:27,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/A/70369733dd5a4510b4d3efa4c80fef98 is 175, key is test_row_0/A:col10/1731278306089/Put/seqid=0 2024-11-10T22:38:27,443 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742071_1247 (size=22561) 2024-11-10T22:38:27,507 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/A/37a20fb724c34fc09699307bedb23540 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/37a20fb724c34fc09699307bedb23540 2024-11-10T22:38:27,519 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d51b53151407a9b9ad82a60ad302f998/A of d51b53151407a9b9ad82a60ad302f998 into 37a20fb724c34fc09699307bedb23540(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T22:38:27,519 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d51b53151407a9b9ad82a60ad302f998: 2024-11-10T22:38:27,519 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., storeName=d51b53151407a9b9ad82a60ad302f998/A, priority=12, startTime=1731278306950; duration=0sec 2024-11-10T22:38:27,519 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T22:38:27,519 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d51b53151407a9b9ad82a60ad302f998:A 2024-11-10T22:38:27,625 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:27,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40738 deadline: 1731278367622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:27,625 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:27,625 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:27,625 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:27,626 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:27,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40754 deadline: 1731278367622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:27,626 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:27,626 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:27,626 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:27,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40736 deadline: 1731278367622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:27,626 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:27,626 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:27,626 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:27,626 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:27,626 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:27,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40774 deadline: 1731278367622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:27,626 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:27,626 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:27,626 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:27,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-10T22:38:27,844 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=389, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/A/70369733dd5a4510b4d3efa4c80fef98 2024-11-10T22:38:27,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/B/c22add54e2b34fd2b2dd30a0b6a8e187 is 50, key is test_row_0/B:col10/1731278306089/Put/seqid=0 2024-11-10T22:38:27,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742072_1248 (size=9857) 2024-11-10T22:38:27,935 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:27,935 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:27,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40738 deadline: 1731278367932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:27,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40736 deadline: 1731278367932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:27,935 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:27,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40754 deadline: 1731278367932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:27,935 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:27,935 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:27,935 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:27,935 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:27,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40774 deadline: 1731278367933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:27,936 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:27,936 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:27,936 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:27,936 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:27,936 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:27,936 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:27,936 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:27,936 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:27,936 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:28,266 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=389 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/B/c22add54e2b34fd2b2dd30a0b6a8e187 2024-11-10T22:38:28,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/C/589d18958c684845a23a18fd20f7acfe is 50, key is test_row_0/C:col10/1731278306089/Put/seqid=0 2024-11-10T22:38:28,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742073_1249 (size=9857) 2024-11-10T22:38:28,444 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:28,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40736 deadline: 1731278368442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:28,445 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:28,445 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:28,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40754 deadline: 1731278368442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:28,445 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:28,445 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:28,445 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:28,445 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:28,445 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:28,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40738 deadline: 1731278368442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:28,445 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:28,445 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:28,445 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:28,445 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:28,445 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:28,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40774 deadline: 1731278368442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:28,445 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:28,445 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:28,445 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:28,682 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=389 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/C/589d18958c684845a23a18fd20f7acfe 2024-11-10T22:38:28,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/A/70369733dd5a4510b4d3efa4c80fef98 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/70369733dd5a4510b4d3efa4c80fef98 2024-11-10T22:38:28,692 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/70369733dd5a4510b4d3efa4c80fef98, entries=100, sequenceid=389, filesize=22.0 K 2024-11-10T22:38:28,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/B/c22add54e2b34fd2b2dd30a0b6a8e187 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B/c22add54e2b34fd2b2dd30a0b6a8e187 2024-11-10T22:38:28,698 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B/c22add54e2b34fd2b2dd30a0b6a8e187, entries=100, sequenceid=389, filesize=9.6 K 2024-11-10T22:38:28,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/C/589d18958c684845a23a18fd20f7acfe as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/C/589d18958c684845a23a18fd20f7acfe 2024-11-10T22:38:28,704 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/C/589d18958c684845a23a18fd20f7acfe, entries=100, sequenceid=389, filesize=9.6 K 2024-11-10T22:38:28,704 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=154.31 KB/158010 for d51b53151407a9b9ad82a60ad302f998 in 1751ms, sequenceid=389, compaction requested=false 2024-11-10T22:38:28,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2603): Flush status journal for d51b53151407a9b9ad82a60ad302f998: 2024-11-10T22:38:28,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. 2024-11-10T22:38:28,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=55 2024-11-10T22:38:28,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.HMaster(4169): Remote procedure done, pid=55 2024-11-10T22:38:28,708 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-11-10T22:38:28,708 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0600 sec 2024-11-10T22:38:28,710 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees in 2.0660 sec 2024-11-10T22:38:28,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-10T22:38:28,792 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T22:38:28,793 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T22:38:28,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] procedure2.ProcedureExecutor(1139): Stored pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees 2024-11-10T22:38:28,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-10T22:38:28,795 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T22:38:28,796 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T22:38:28,796 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T22:38:28,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-10T22:38:28,948 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-10T22:38:28,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. 2024-11-10T22:38:28,948 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2902): Flushing d51b53151407a9b9ad82a60ad302f998 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-10T22:38:28,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110dd915cd70f094b6b8901efdfd76cdcc1_d51b53151407a9b9ad82a60ad302f998 is 50, key is test_row_0/A:col10/1731278307303/Put/seqid=0 2024-11-10T22:38:28,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742074_1250 (size=12454) 2024-11-10T22:38:29,063 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-10T22:38:29,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-10T22:38:29,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,367 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110dd915cd70f094b6b8901efdfd76cdcc1_d51b53151407a9b9ad82a60ad302f998 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110dd915cd70f094b6b8901efdfd76cdcc1_d51b53151407a9b9ad82a60ad302f998 2024-11-10T22:38:29,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/A/e9ea78ce2b9147edaff0ce52c4cfb11f, store: [table=TestAcidGuarantees family=A region=d51b53151407a9b9ad82a60ad302f998] 2024-11-10T22:38:29,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/A/e9ea78ce2b9147edaff0ce52c4cfb11f is 175, key is test_row_0/A:col10/1731278307303/Put/seqid=0 2024-11-10T22:38:29,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742075_1251 (size=31255) 2024-11-10T22:38:29,377 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=418, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/A/e9ea78ce2b9147edaff0ce52c4cfb11f 2024-11-10T22:38:29,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/B/9c453f92666740eeba459d59cfc085d6 is 50, key is test_row_0/B:col10/1731278307303/Put/seqid=0 2024-11-10T22:38:29,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742076_1252 (size=12301) 2024-11-10T22:38:29,391 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=418 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/B/9c453f92666740eeba459d59cfc085d6 2024-11-10T22:38:29,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/C/ee13695e442744f6b184bdd7bf010c58 is 50, key is test_row_0/C:col10/1731278307303/Put/seqid=0 2024-11-10T22:38:29,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742077_1253 (size=12301) 2024-11-10T22:38:29,405 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=418 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/C/ee13695e442744f6b184bdd7bf010c58 2024-11-10T22:38:29,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/A/e9ea78ce2b9147edaff0ce52c4cfb11f as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/e9ea78ce2b9147edaff0ce52c4cfb11f 2024-11-10T22:38:29,413 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/e9ea78ce2b9147edaff0ce52c4cfb11f, entries=150, sequenceid=418, filesize=30.5 K 2024-11-10T22:38:29,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/B/9c453f92666740eeba459d59cfc085d6 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B/9c453f92666740eeba459d59cfc085d6 2024-11-10T22:38:29,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,418 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B/9c453f92666740eeba459d59cfc085d6, entries=150, sequenceid=418, filesize=12.0 K 2024-11-10T22:38:29,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/C/ee13695e442744f6b184bdd7bf010c58 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/C/ee13695e442744f6b184bdd7bf010c58 2024-11-10T22:38:29,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,424 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/C/ee13695e442744f6b184bdd7bf010c58, entries=150, sequenceid=418, filesize=12.0 K 2024-11-10T22:38:29,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,425 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=0 B/0 for d51b53151407a9b9ad82a60ad302f998 in 477ms, sequenceid=418, compaction requested=true 2024-11-10T22:38:29,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2603): Flush status journal for d51b53151407a9b9ad82a60ad302f998: 2024-11-10T22:38:29,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. 2024-11-10T22:38:29,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=57 2024-11-10T22:38:29,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.HMaster(4169): Remote procedure done, pid=57 2024-11-10T22:38:29,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-10T22:38:29,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,429 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-11-10T22:38:29,429 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 631 msec 2024-11-10T22:38:29,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,431 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees in 637 msec 2024-11-10T22:38:29,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,511 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T22:38:29,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,511 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d51b53151407a9b9ad82a60ad302f998 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-10T22:38:29,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(8855): Flush requested on d51b53151407a9b9ad82a60ad302f998 2024-11-10T22:38:29,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,526 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411106df971e0fed34b7f9b003b3de878989c_d51b53151407a9b9ad82a60ad302f998 is 50, key is test_row_0/A:col10/1731278309509/Put/seqid=0 2024-11-10T22:38:29,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742079_1255 (size=25158) 2024-11-10T22:38:29,537 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,542 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411106df971e0fed34b7f9b003b3de878989c_d51b53151407a9b9ad82a60ad302f998 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411106df971e0fed34b7f9b003b3de878989c_d51b53151407a9b9ad82a60ad302f998 2024-11-10T22:38:29,543 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/A/656e2f90b48b46d59ba0e39ffdc8e44c, store: [table=TestAcidGuarantees family=A region=d51b53151407a9b9ad82a60ad302f998] 2024-11-10T22:38:29,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,544 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/A/656e2f90b48b46d59ba0e39ffdc8e44c is 175, key is test_row_0/A:col10/1731278309509/Put/seqid=0 2024-11-10T22:38:29,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742078_1254 (size=74795) 2024-11-10T22:38:29,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,547 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=429, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/A/656e2f90b48b46d59ba0e39ffdc8e44c 2024-11-10T22:38:29,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,559 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/B/a984f7f7264d48de87eec493b356ebaf is 50, key is test_row_0/B:col10/1731278309509/Put/seqid=0 2024-11-10T22:38:29,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742080_1256 (size=24505) 2024-11-10T22:38:29,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,578 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:29,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40736 deadline: 1731278369571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:29,578 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:29,578 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:29,578 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:29,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,578 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:29,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40754 deadline: 1731278369571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:29,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,579 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:29,579 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:29,579 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:29,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:29,584 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:29,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40774 deadline: 1731278369578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:29,585 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:29,585 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:29,585 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:29,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40738 deadline: 1731278369578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:29,585 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:29,585 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:29,585 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:29,585 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:29,684 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:29,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40736 deadline: 1731278369683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:29,685 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:29,685 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:29,685 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:29,685 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:29,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40754 deadline: 1731278369683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:29,685 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:29,685 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:29,685 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:29,695 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:29,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40774 deadline: 1731278369692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:29,696 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:29,696 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:29,696 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:29,696 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:29,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40738 deadline: 1731278369692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:29,696 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:29,696 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:29,696 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:29,832 INFO [Thread-698 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-698. 2024-11-10T22:38:29,832 DEBUG [Thread-698 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T22:38:29,832 DEBUG [Thread-698 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:38:29,832 DEBUG [Thread-698 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:38:29,832 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T22:38:29,834 INFO [Thread-702 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-702. 2024-11-10T22:38:29,834 DEBUG [Thread-702 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T22:38:29,834 DEBUG [Thread-702 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:38:29,834 DEBUG [Thread-702 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:38:29,834 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T22:38:29,835 INFO [Thread-696 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-696. 2024-11-10T22:38:29,835 DEBUG [Thread-696 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T22:38:29,835 DEBUG [Thread-696 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:38:29,835 INFO [Thread-700 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-700. 2024-11-10T22:38:29,835 DEBUG [Thread-700 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T22:38:29,835 DEBUG [Thread-700 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:38:29,835 DEBUG [Thread-696 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:38:29,836 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T22:38:29,836 DEBUG [Thread-700 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:38:29,836 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T22:38:29,893 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:29,893 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:29,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40736 deadline: 1731278369893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:29,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40754 deadline: 1731278369893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:29,893 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:29,893 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:29,893 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:29,893 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:29,893 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:29,893 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:29,902 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:29,902 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:29,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40774 deadline: 1731278369902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:29,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40738 deadline: 1731278369902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:29,902 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:29,902 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:29,902 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:29,902 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:29,902 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:29,902 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:29,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-10T22:38:29,933 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T22:38:29,966 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=429 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/B/a984f7f7264d48de87eec493b356ebaf 2024-11-10T22:38:29,973 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/C/395a96605ccb4a8ca6eeb5eb51ae0c40 is 50, key is test_row_0/C:col10/1731278309509/Put/seqid=0 2024-11-10T22:38:29,976 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742081_1257 (size=12301) 2024-11-10T22:38:30,202 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:30,202 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:30,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40754 deadline: 1731278370202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:30,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40736 deadline: 1731278370202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:30,203 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:30,203 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:30,203 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:30,203 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:30,203 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:30,203 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:30,213 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:30,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40774 deadline: 1731278370212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:30,213 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:30,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40738 deadline: 1731278370212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:30,213 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:30,213 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:30,213 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:30,213 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:30,213 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:30,213 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:38:30,377 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=429 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/C/395a96605ccb4a8ca6eeb5eb51ae0c40 2024-11-10T22:38:30,382 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/A/656e2f90b48b46d59ba0e39ffdc8e44c as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/656e2f90b48b46d59ba0e39ffdc8e44c 2024-11-10T22:38:30,386 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/656e2f90b48b46d59ba0e39ffdc8e44c, entries=400, sequenceid=429, filesize=73.0 K 2024-11-10T22:38:30,387 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/B/a984f7f7264d48de87eec493b356ebaf as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B/a984f7f7264d48de87eec493b356ebaf 2024-11-10T22:38:30,390 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B/a984f7f7264d48de87eec493b356ebaf, entries=400, sequenceid=429, filesize=23.9 K 2024-11-10T22:38:30,391 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/C/395a96605ccb4a8ca6eeb5eb51ae0c40 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/C/395a96605ccb4a8ca6eeb5eb51ae0c40 2024-11-10T22:38:30,394 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/C/395a96605ccb4a8ca6eeb5eb51ae0c40, entries=150, sequenceid=429, filesize=12.0 K 2024-11-10T22:38:30,395 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for d51b53151407a9b9ad82a60ad302f998 in 884ms, sequenceid=429, compaction requested=true 2024-11-10T22:38:30,395 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d51b53151407a9b9ad82a60ad302f998: 2024-11-10T22:38:30,395 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d51b53151407a9b9ad82a60ad302f998:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T22:38:30,395 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T22:38:30,395 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d51b53151407a9b9ad82a60ad302f998:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T22:38:30,395 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T22:38:30,395 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d51b53151407a9b9ad82a60ad302f998:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T22:38:30,395 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T22:38:30,395 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T22:38:30,395 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T22:38:30,396 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 59850 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T22:38:30,396 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 160752 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T22:38:30,396 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HStore(1541): d51b53151407a9b9ad82a60ad302f998/A is initiating minor compaction (all files) 2024-11-10T22:38:30,396 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1541): d51b53151407a9b9ad82a60ad302f998/B is initiating minor compaction (all files) 2024-11-10T22:38:30,396 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d51b53151407a9b9ad82a60ad302f998/A in TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. 2024-11-10T22:38:30,396 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d51b53151407a9b9ad82a60ad302f998/B in TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. 2024-11-10T22:38:30,397 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/37a20fb724c34fc09699307bedb23540, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/70369733dd5a4510b4d3efa4c80fef98, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/e9ea78ce2b9147edaff0ce52c4cfb11f, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/656e2f90b48b46d59ba0e39ffdc8e44c] into tmpdir=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp, totalSize=157.0 K 2024-11-10T22:38:30,397 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B/675f97d30adc4ae7827e9deed06ed49b, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B/c22add54e2b34fd2b2dd30a0b6a8e187, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B/9c453f92666740eeba459d59cfc085d6, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B/a984f7f7264d48de87eec493b356ebaf] into tmpdir=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp, totalSize=58.4 K 2024-11-10T22:38:30,397 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. 2024-11-10T22:38:30,397 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. files: [hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/37a20fb724c34fc09699307bedb23540, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/70369733dd5a4510b4d3efa4c80fef98, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/e9ea78ce2b9147edaff0ce52c4cfb11f, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/656e2f90b48b46d59ba0e39ffdc8e44c] 2024-11-10T22:38:30,397 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting 675f97d30adc4ae7827e9deed06ed49b, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=379, earliestPutTs=1731278303920 2024-11-10T22:38:30,397 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 37a20fb724c34fc09699307bedb23540, keycount=150, bloomtype=ROW, size=31.4 K, encoding=NONE, compression=NONE, seqNum=379, earliestPutTs=1731278303920 2024-11-10T22:38:30,397 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting c22add54e2b34fd2b2dd30a0b6a8e187, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=389, earliestPutTs=1731278306080 2024-11-10T22:38:30,397 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 70369733dd5a4510b4d3efa4c80fef98, keycount=100, bloomtype=ROW, size=22.0 K, encoding=NONE, compression=NONE, seqNum=389, earliestPutTs=1731278306080 2024-11-10T22:38:30,397 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting 9c453f92666740eeba459d59cfc085d6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=418, earliestPutTs=1731278307298 2024-11-10T22:38:30,397 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting e9ea78ce2b9147edaff0ce52c4cfb11f, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=418, earliestPutTs=1731278307298 2024-11-10T22:38:30,398 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting a984f7f7264d48de87eec493b356ebaf, keycount=400, bloomtype=ROW, size=23.9 K, encoding=NONE, compression=NONE, seqNum=429, earliestPutTs=1731278309455 2024-11-10T22:38:30,398 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 656e2f90b48b46d59ba0e39ffdc8e44c, keycount=400, bloomtype=ROW, size=73.0 K, encoding=NONE, compression=NONE, seqNum=429, earliestPutTs=1731278309455 2024-11-10T22:38:30,405 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d51b53151407a9b9ad82a60ad302f998] 2024-11-10T22:38:30,407 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411106ef53283bebb4831b3c4a5b94a9e7d64_d51b53151407a9b9ad82a60ad302f998 store=[table=TestAcidGuarantees family=A region=d51b53151407a9b9ad82a60ad302f998] 2024-11-10T22:38:30,429 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d51b53151407a9b9ad82a60ad302f998#B#compaction#213 average throughput is 0.27 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T22:38:30,430 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/B/567d986d313049d3b8017e6d8af309f9 is 50, key is test_row_0/B:col10/1731278309509/Put/seqid=0 2024-11-10T22:38:30,433 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411106ef53283bebb4831b3c4a5b94a9e7d64_d51b53151407a9b9ad82a60ad302f998, store=[table=TestAcidGuarantees family=A region=d51b53151407a9b9ad82a60ad302f998] 2024-11-10T22:38:30,433 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411106ef53283bebb4831b3c4a5b94a9e7d64_d51b53151407a9b9ad82a60ad302f998 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d51b53151407a9b9ad82a60ad302f998] 2024-11-10T22:38:30,437 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742082_1258 (size=13323) 2024-11-10T22:38:30,445 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/B/567d986d313049d3b8017e6d8af309f9 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B/567d986d313049d3b8017e6d8af309f9 2024-11-10T22:38:30,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742083_1259 (size=4469) 2024-11-10T22:38:30,451 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d51b53151407a9b9ad82a60ad302f998/B of d51b53151407a9b9ad82a60ad302f998 into 567d986d313049d3b8017e6d8af309f9(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T22:38:30,451 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d51b53151407a9b9ad82a60ad302f998: 2024-11-10T22:38:30,451 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d51b53151407a9b9ad82a60ad302f998#A#compaction#212 average throughput is 0.53 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T22:38:30,451 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., storeName=d51b53151407a9b9ad82a60ad302f998/B, priority=12, startTime=1731278310395; duration=0sec 2024-11-10T22:38:30,451 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T22:38:30,451 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d51b53151407a9b9ad82a60ad302f998:B 2024-11-10T22:38:30,451 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T22:38:30,452 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/A/5d9243e961404207b5c604a350a33733 is 175, key is test_row_0/A:col10/1731278309509/Put/seqid=0 2024-11-10T22:38:30,453 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47612 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T22:38:30,453 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1541): d51b53151407a9b9ad82a60ad302f998/C is initiating minor compaction (all files) 2024-11-10T22:38:30,454 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d51b53151407a9b9ad82a60ad302f998/C in TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. 2024-11-10T22:38:30,454 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/C/e4ed274f3413411984807705284474c2, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/C/589d18958c684845a23a18fd20f7acfe, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/C/ee13695e442744f6b184bdd7bf010c58, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/C/395a96605ccb4a8ca6eeb5eb51ae0c40] into tmpdir=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp, totalSize=46.5 K 2024-11-10T22:38:30,454 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting e4ed274f3413411984807705284474c2, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=379, earliestPutTs=1731278303920 2024-11-10T22:38:30,454 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting 589d18958c684845a23a18fd20f7acfe, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=389, earliestPutTs=1731278306080 2024-11-10T22:38:30,455 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting ee13695e442744f6b184bdd7bf010c58, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=418, earliestPutTs=1731278307298 2024-11-10T22:38:30,455 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting 395a96605ccb4a8ca6eeb5eb51ae0c40, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=429, earliestPutTs=1731278309503 2024-11-10T22:38:30,456 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742084_1260 (size=32277) 2024-11-10T22:38:30,460 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/A/5d9243e961404207b5c604a350a33733 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/5d9243e961404207b5c604a350a33733 2024-11-10T22:38:30,465 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d51b53151407a9b9ad82a60ad302f998/A of d51b53151407a9b9ad82a60ad302f998 into 5d9243e961404207b5c604a350a33733(size=31.5 K), total size for store is 31.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T22:38:30,465 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d51b53151407a9b9ad82a60ad302f998: 2024-11-10T22:38:30,465 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., storeName=d51b53151407a9b9ad82a60ad302f998/A, priority=12, startTime=1731278310395; duration=0sec 2024-11-10T22:38:30,465 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T22:38:30,465 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d51b53151407a9b9ad82a60ad302f998:A 2024-11-10T22:38:30,469 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d51b53151407a9b9ad82a60ad302f998#C#compaction#214 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T22:38:30,469 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/C/2d1230aa357a4571a1d54879cb1fad36 is 50, key is test_row_0/C:col10/1731278309509/Put/seqid=0 2024-11-10T22:38:30,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742085_1261 (size=13289) 2024-11-10T22:38:30,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(8855): Flush requested on d51b53151407a9b9ad82a60ad302f998 2024-11-10T22:38:30,499 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T22:38:30,499 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d51b53151407a9b9ad82a60ad302f998 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-10T22:38:30,499 INFO [Thread-685 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-685. 2024-11-10T22:38:30,499 DEBUG [Thread-685 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T22:38:30,499 DEBUG [Thread-685 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:38:30,499 DEBUG [Thread-685 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:38:30,499 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T22:38:30,511 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411104e16e1c38a194082a44abcec6cfb7a57_d51b53151407a9b9ad82a60ad302f998 is 50, key is test_row_0/A:col10/1731278309576/Put/seqid=0 2024-11-10T22:38:30,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742086_1262 (size=12454) 2024-11-10T22:38:30,714 INFO [Thread-693 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-693. 2024-11-10T22:38:30,714 INFO [Thread-689 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-689. 2024-11-10T22:38:30,714 DEBUG [Thread-689 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T22:38:30,714 DEBUG [Thread-693 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T22:38:30,714 DEBUG [Thread-689 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:38:30,714 DEBUG [Thread-693 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:38:30,714 DEBUG [Thread-693 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:38:30,714 DEBUG [Thread-689 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:38:30,714 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T22:38:30,714 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T22:38:30,724 INFO [Thread-691 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-691. 2024-11-10T22:38:30,724 INFO [Thread-687 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-687. 2024-11-10T22:38:30,724 DEBUG [Thread-691 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T22:38:30,724 DEBUG [Thread-687 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T22:38:30,724 DEBUG [Thread-691 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:38:30,724 DEBUG [Thread-687 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:38:30,724 DEBUG [Thread-691 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:38:30,724 DEBUG [Thread-687 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:38:30,724 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T22:38:30,724 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T22:38:30,724 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-10T22:38:30,725 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 87 2024-11-10T22:38:30,725 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 70 2024-11-10T22:38:30,725 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 66 2024-11-10T22:38:30,725 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 73 2024-11-10T22:38:30,725 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 67 2024-11-10T22:38:30,725 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-10T22:38:30,725 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3408 2024-11-10T22:38:30,725 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3503 2024-11-10T22:38:30,725 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-10T22:38:30,725 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1494 2024-11-10T22:38:30,725 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4482 rows 2024-11-10T22:38:30,725 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1497 2024-11-10T22:38:30,725 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4491 rows 2024-11-10T22:38:30,725 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-10T22:38:30,725 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T22:38:30,725 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:38:30,725 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:38:30,725 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T22:38:30,726 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-10T22:38:30,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] procedure2.ProcedureExecutor(1139): Stored pid=58, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-10T22:38:30,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-10T22:38:30,729 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731278310729"}]},"ts":"1731278310729"} 2024-11-10T22:38:30,730 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-10T22:38:30,730 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-10T22:38:30,731 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-10T22:38:30,732 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d51b53151407a9b9ad82a60ad302f998, UNASSIGN}] 2024-11-10T22:38:30,733 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d51b53151407a9b9ad82a60ad302f998, UNASSIGN 2024-11-10T22:38:30,733 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=d51b53151407a9b9ad82a60ad302f998, regionState=CLOSING, regionLocation=849d0d71310f,43753,1731278250663 2024-11-10T22:38:30,734 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d51b53151407a9b9ad82a60ad302f998, UNASSIGN because future has completed 2024-11-10T22:38:30,735 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-10T22:38:30,735 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; CloseRegionProcedure d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663}] 2024-11-10T22:38:30,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-10T22:38:30,878 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/C/2d1230aa357a4571a1d54879cb1fad36 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/C/2d1230aa357a4571a1d54879cb1fad36 2024-11-10T22:38:30,883 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d51b53151407a9b9ad82a60ad302f998/C of d51b53151407a9b9ad82a60ad302f998 into 2d1230aa357a4571a1d54879cb1fad36(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T22:38:30,883 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d51b53151407a9b9ad82a60ad302f998: 2024-11-10T22:38:30,883 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., storeName=d51b53151407a9b9ad82a60ad302f998/C, priority=12, startTime=1731278310395; duration=0sec 2024-11-10T22:38:30,883 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T22:38:30,883 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d51b53151407a9b9ad82a60ad302f998:C 2024-11-10T22:38:30,887 INFO [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] handler.UnassignRegionHandler(122): Close d51b53151407a9b9ad82a60ad302f998 2024-11-10T22:38:30,887 DEBUG [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-10T22:38:30,887 DEBUG [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1722): Closing d51b53151407a9b9ad82a60ad302f998, disabling compactions & flushes 2024-11-10T22:38:30,887 DEBUG [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. 2024-11-10T22:38:30,916 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:30,919 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411104e16e1c38a194082a44abcec6cfb7a57_d51b53151407a9b9ad82a60ad302f998 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411104e16e1c38a194082a44abcec6cfb7a57_d51b53151407a9b9ad82a60ad302f998 2024-11-10T22:38:30,920 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/A/6374c379f3844db98663f6fcb766aed4, store: [table=TestAcidGuarantees family=A region=d51b53151407a9b9ad82a60ad302f998] 2024-11-10T22:38:30,920 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/A/6374c379f3844db98663f6fcb766aed4 is 175, key is test_row_0/A:col10/1731278309576/Put/seqid=0 2024-11-10T22:38:30,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742087_1263 (size=31255) 2024-11-10T22:38:31,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-10T22:38:31,324 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=457, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/A/6374c379f3844db98663f6fcb766aed4 2024-11-10T22:38:31,331 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/B/aaaaf6420e944b4ca3ba3b3f030a3346 is 50, key is test_row_0/B:col10/1731278309576/Put/seqid=0 2024-11-10T22:38:31,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742088_1264 (size=12301) 2024-11-10T22:38:31,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-10T22:38:31,736 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=457 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/B/aaaaf6420e944b4ca3ba3b3f030a3346 2024-11-10T22:38:31,742 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/C/9de52c9fbc4447a59d2a0f201f4f4b54 is 50, key is test_row_0/C:col10/1731278309576/Put/seqid=0 2024-11-10T22:38:31,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742089_1265 (size=12301) 2024-11-10T22:38:31,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-10T22:38:32,147 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=457 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/C/9de52c9fbc4447a59d2a0f201f4f4b54 2024-11-10T22:38:32,150 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/A/6374c379f3844db98663f6fcb766aed4 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/6374c379f3844db98663f6fcb766aed4 2024-11-10T22:38:32,153 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/6374c379f3844db98663f6fcb766aed4, entries=150, sequenceid=457, filesize=30.5 K 2024-11-10T22:38:32,154 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/B/aaaaf6420e944b4ca3ba3b3f030a3346 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B/aaaaf6420e944b4ca3ba3b3f030a3346 2024-11-10T22:38:32,157 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B/aaaaf6420e944b4ca3ba3b3f030a3346, entries=150, sequenceid=457, filesize=12.0 K 2024-11-10T22:38:32,158 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/C/9de52c9fbc4447a59d2a0f201f4f4b54 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/C/9de52c9fbc4447a59d2a0f201f4f4b54 2024-11-10T22:38:32,161 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/C/9de52c9fbc4447a59d2a0f201f4f4b54, entries=150, sequenceid=457, filesize=12.0 K 2024-11-10T22:38:32,161 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=26.84 KB/27480 for d51b53151407a9b9ad82a60ad302f998 in 1662ms, sequenceid=457, compaction requested=false 2024-11-10T22:38:32,161 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d51b53151407a9b9ad82a60ad302f998: 2024-11-10T22:38:32,162 INFO [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. 2024-11-10T22:38:32,162 DEBUG [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. 2024-11-10T22:38:32,162 DEBUG [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. after waiting 0 ms 2024-11-10T22:38:32,162 DEBUG [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. 2024-11-10T22:38:32,162 INFO [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(2902): Flushing d51b53151407a9b9ad82a60ad302f998 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2024-11-10T22:38:32,167 DEBUG [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411107a7f38519ecc499eb4c60ce713bf21ac_d51b53151407a9b9ad82a60ad302f998 is 50, key is test_row_0/A:col10/1731278310723/Put/seqid=0 2024-11-10T22:38:32,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742090_1266 (size=12454) 2024-11-10T22:38:32,571 DEBUG [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:32,574 INFO [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411107a7f38519ecc499eb4c60ce713bf21ac_d51b53151407a9b9ad82a60ad302f998 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411107a7f38519ecc499eb4c60ce713bf21ac_d51b53151407a9b9ad82a60ad302f998 2024-11-10T22:38:32,575 DEBUG [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/A/220131f58ea94bef8718d1aa9f45fe7b, store: [table=TestAcidGuarantees family=A region=d51b53151407a9b9ad82a60ad302f998] 2024-11-10T22:38:32,576 DEBUG [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/A/220131f58ea94bef8718d1aa9f45fe7b is 175, key is test_row_0/A:col10/1731278310723/Put/seqid=0 2024-11-10T22:38:32,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742091_1267 (size=31255) 2024-11-10T22:38:32,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-10T22:38:32,980 INFO [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=465, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/A/220131f58ea94bef8718d1aa9f45fe7b 2024-11-10T22:38:32,986 DEBUG [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/B/7b0044e869ae4b19b1c57aa9f6f82f3e is 50, key is test_row_0/B:col10/1731278310723/Put/seqid=0 2024-11-10T22:38:32,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742092_1268 (size=12301) 2024-11-10T22:38:33,390 INFO [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=465 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/B/7b0044e869ae4b19b1c57aa9f6f82f3e 2024-11-10T22:38:33,396 DEBUG [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/C/53c9754c4bd448f6823c4c2d39712cdb is 50, key is test_row_0/C:col10/1731278310723/Put/seqid=0 2024-11-10T22:38:33,400 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742093_1269 (size=12301) 2024-11-10T22:38:33,800 INFO [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=465 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/C/53c9754c4bd448f6823c4c2d39712cdb 2024-11-10T22:38:33,805 DEBUG [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/A/220131f58ea94bef8718d1aa9f45fe7b as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/220131f58ea94bef8718d1aa9f45fe7b 2024-11-10T22:38:33,809 INFO [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/220131f58ea94bef8718d1aa9f45fe7b, entries=150, sequenceid=465, filesize=30.5 K 2024-11-10T22:38:33,809 DEBUG [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/B/7b0044e869ae4b19b1c57aa9f6f82f3e as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B/7b0044e869ae4b19b1c57aa9f6f82f3e 2024-11-10T22:38:33,813 INFO [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B/7b0044e869ae4b19b1c57aa9f6f82f3e, entries=150, sequenceid=465, filesize=12.0 K 2024-11-10T22:38:33,813 DEBUG [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/.tmp/C/53c9754c4bd448f6823c4c2d39712cdb as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/C/53c9754c4bd448f6823c4c2d39712cdb 2024-11-10T22:38:33,816 INFO [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/C/53c9754c4bd448f6823c4c2d39712cdb, entries=150, sequenceid=465, filesize=12.0 K 2024-11-10T22:38:33,817 INFO [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for d51b53151407a9b9ad82a60ad302f998 in 1655ms, sequenceid=465, compaction requested=true 2024-11-10T22:38:33,818 DEBUG [StoreCloser-TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/cfcb2c04057f42798b3273500fe63c42, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/85b308ab4be1446892c81e8011f69e87, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/6a77243a66f447039c7e1c471eb3a4aa, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/a21a3de589ea499aadc8772b3865ef6c, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/099cb422e84c476db3a97644ce788de9, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/16860bcb95a34b6487878236e7302a1c, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/9036257355fb403bb6ed4fc7ce5b9efd, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/896a89738257408ab02f7a4df7b0bac9, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/8810867aca3b42af8c217923a96327aa, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/a5d1a25527f24defaea12e3250b0e326, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/359698e468ba47788d1b67fe67724389, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/3c53ea79ea674e708881fdc7007f15ac, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/4c31db80c94b46a18023ecf1c2adf870, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/5bfa0d7c3e0547e3933078168f04ea8a, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/3b7faa6de25f4d35bd1ce967d3f2d017, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/1e4c9970088f487d8bee0e437576b116, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/34133856ee764d95ab9341c48b9ee49b, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/8c0efc7b662e40fb8f682f6cb9fcd6ff, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/05125de77e6540978f99ea1243b2d47c, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/788648d6317e475d9691263443b8e150, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/adc130dbc8e448bd8bce9f694c7e9860, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/518cff827f854a1d89d1822bc9ad8b51, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/f9c3d2b4688e4cfd8ec2dc98bdadd648, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/03ee732a8ba04a35a098f9c3d835945b, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/ee8f97c1d0754baa8271a277d811c0a7, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/5f1dac635c454fe0bee47e308dfed044, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/37a20fb724c34fc09699307bedb23540, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/70369733dd5a4510b4d3efa4c80fef98, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/e9ea78ce2b9147edaff0ce52c4cfb11f, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/656e2f90b48b46d59ba0e39ffdc8e44c] to archive 2024-11-10T22:38:33,819 DEBUG [StoreCloser-TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T22:38:33,820 DEBUG [StoreCloser-TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/cfcb2c04057f42798b3273500fe63c42 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/cfcb2c04057f42798b3273500fe63c42 2024-11-10T22:38:33,821 DEBUG [StoreCloser-TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/85b308ab4be1446892c81e8011f69e87 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/85b308ab4be1446892c81e8011f69e87 2024-11-10T22:38:33,822 DEBUG [StoreCloser-TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/6a77243a66f447039c7e1c471eb3a4aa to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/6a77243a66f447039c7e1c471eb3a4aa 2024-11-10T22:38:33,823 DEBUG [StoreCloser-TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/a21a3de589ea499aadc8772b3865ef6c to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/a21a3de589ea499aadc8772b3865ef6c 2024-11-10T22:38:33,824 DEBUG [StoreCloser-TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/099cb422e84c476db3a97644ce788de9 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/099cb422e84c476db3a97644ce788de9 2024-11-10T22:38:33,825 DEBUG [StoreCloser-TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/16860bcb95a34b6487878236e7302a1c to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/16860bcb95a34b6487878236e7302a1c 2024-11-10T22:38:33,826 DEBUG [StoreCloser-TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/9036257355fb403bb6ed4fc7ce5b9efd to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/9036257355fb403bb6ed4fc7ce5b9efd 2024-11-10T22:38:33,827 DEBUG [StoreCloser-TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/896a89738257408ab02f7a4df7b0bac9 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/896a89738257408ab02f7a4df7b0bac9 2024-11-10T22:38:33,828 DEBUG [StoreCloser-TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/8810867aca3b42af8c217923a96327aa to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/8810867aca3b42af8c217923a96327aa 2024-11-10T22:38:33,829 DEBUG [StoreCloser-TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/a5d1a25527f24defaea12e3250b0e326 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/a5d1a25527f24defaea12e3250b0e326 2024-11-10T22:38:33,830 DEBUG [StoreCloser-TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/359698e468ba47788d1b67fe67724389 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/359698e468ba47788d1b67fe67724389 2024-11-10T22:38:33,831 DEBUG [StoreCloser-TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/3c53ea79ea674e708881fdc7007f15ac to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/3c53ea79ea674e708881fdc7007f15ac 2024-11-10T22:38:33,832 DEBUG [StoreCloser-TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/4c31db80c94b46a18023ecf1c2adf870 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/4c31db80c94b46a18023ecf1c2adf870 2024-11-10T22:38:33,833 DEBUG [StoreCloser-TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/5bfa0d7c3e0547e3933078168f04ea8a to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/5bfa0d7c3e0547e3933078168f04ea8a 2024-11-10T22:38:33,834 DEBUG [StoreCloser-TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/3b7faa6de25f4d35bd1ce967d3f2d017 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/3b7faa6de25f4d35bd1ce967d3f2d017 2024-11-10T22:38:33,835 DEBUG [StoreCloser-TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/1e4c9970088f487d8bee0e437576b116 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/1e4c9970088f487d8bee0e437576b116 2024-11-10T22:38:33,836 DEBUG [StoreCloser-TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/34133856ee764d95ab9341c48b9ee49b to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/34133856ee764d95ab9341c48b9ee49b 2024-11-10T22:38:33,837 DEBUG [StoreCloser-TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/8c0efc7b662e40fb8f682f6cb9fcd6ff to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/8c0efc7b662e40fb8f682f6cb9fcd6ff 2024-11-10T22:38:33,838 DEBUG [StoreCloser-TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/05125de77e6540978f99ea1243b2d47c to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/05125de77e6540978f99ea1243b2d47c 2024-11-10T22:38:33,839 DEBUG [StoreCloser-TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/788648d6317e475d9691263443b8e150 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/788648d6317e475d9691263443b8e150 2024-11-10T22:38:33,840 DEBUG [StoreCloser-TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/adc130dbc8e448bd8bce9f694c7e9860 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/adc130dbc8e448bd8bce9f694c7e9860 2024-11-10T22:38:33,841 DEBUG [StoreCloser-TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/518cff827f854a1d89d1822bc9ad8b51 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/518cff827f854a1d89d1822bc9ad8b51 2024-11-10T22:38:33,842 DEBUG [StoreCloser-TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/f9c3d2b4688e4cfd8ec2dc98bdadd648 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/f9c3d2b4688e4cfd8ec2dc98bdadd648 2024-11-10T22:38:33,843 DEBUG [StoreCloser-TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/03ee732a8ba04a35a098f9c3d835945b to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/03ee732a8ba04a35a098f9c3d835945b 2024-11-10T22:38:33,844 DEBUG [StoreCloser-TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/ee8f97c1d0754baa8271a277d811c0a7 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/ee8f97c1d0754baa8271a277d811c0a7 2024-11-10T22:38:33,845 DEBUG [StoreCloser-TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/5f1dac635c454fe0bee47e308dfed044 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/5f1dac635c454fe0bee47e308dfed044 2024-11-10T22:38:33,846 DEBUG [StoreCloser-TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/37a20fb724c34fc09699307bedb23540 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/37a20fb724c34fc09699307bedb23540 2024-11-10T22:38:33,847 DEBUG [StoreCloser-TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/70369733dd5a4510b4d3efa4c80fef98 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/70369733dd5a4510b4d3efa4c80fef98 2024-11-10T22:38:33,848 DEBUG [StoreCloser-TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/e9ea78ce2b9147edaff0ce52c4cfb11f to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/e9ea78ce2b9147edaff0ce52c4cfb11f 2024-11-10T22:38:33,849 DEBUG [StoreCloser-TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/656e2f90b48b46d59ba0e39ffdc8e44c to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/656e2f90b48b46d59ba0e39ffdc8e44c 2024-11-10T22:38:33,851 DEBUG [StoreCloser-TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B/8629a25e9fb34d728400b1dd1d2f8f1d, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B/a0127e246772431f8a0317025a507e44, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B/d42420e675a24a4e83fc614cce54cbba, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B/8bc9bce4fe03423e839ff893e172c332, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B/13698c4362374b62912bd5df301b5fa9, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B/ad445e689641457dbfbd04005208429c, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B/97e2bd2d217443348f4734caba5f6c63, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B/33835994db69410ea57ea2a9ce009173, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B/79f0d3719e57470d8fb01c738594f277, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B/db2538418a264226914ded5758134157, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B/c47796a3673f4344bbee360f081a9433, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B/83e5a514e8b04664aa895cb8f7c55ff3, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B/486fc2c33da9494f8fc3be85f64d8c5c, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B/d38e66981d7c4aba9163521b3bc9196b, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B/561a785c66bc46d795bd14b7071fd4ab, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B/34549fb4911140f0bbd6df8c7ea97cc0, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B/d7c0f2f7d5e541a5a942cd011800c1bb, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B/6176c8a97b074e45aeb42701dd1e9ec4, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B/d01eb6c178c64462b69c963b493841bb, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B/d8609fed597b41d4bd27515f92539596, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B/e34ba7aebece45968f384f5a064604fc, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B/08ce541f4a6840e5bfb9a2d8ff77bdaf, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B/117a94cb724d40808ca895e4e3d80ffa, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B/b67e9ac5e9fd4995a8ad930ee9082032, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B/21608d1a1e6e484699ee3b75ebbe7da2, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B/675f97d30adc4ae7827e9deed06ed49b, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B/95e1dcb92203407f8d6728e0a413675a, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B/c22add54e2b34fd2b2dd30a0b6a8e187, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B/9c453f92666740eeba459d59cfc085d6, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B/a984f7f7264d48de87eec493b356ebaf] to archive 2024-11-10T22:38:33,851 DEBUG [StoreCloser-TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T22:38:33,853 DEBUG [StoreCloser-TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B/8629a25e9fb34d728400b1dd1d2f8f1d to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B/8629a25e9fb34d728400b1dd1d2f8f1d 2024-11-10T22:38:33,854 DEBUG [StoreCloser-TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B/a0127e246772431f8a0317025a507e44 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B/a0127e246772431f8a0317025a507e44 2024-11-10T22:38:33,855 DEBUG [StoreCloser-TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B/d42420e675a24a4e83fc614cce54cbba to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B/d42420e675a24a4e83fc614cce54cbba 2024-11-10T22:38:33,856 DEBUG [StoreCloser-TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B/8bc9bce4fe03423e839ff893e172c332 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B/8bc9bce4fe03423e839ff893e172c332 2024-11-10T22:38:33,857 DEBUG [StoreCloser-TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B/13698c4362374b62912bd5df301b5fa9 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B/13698c4362374b62912bd5df301b5fa9 2024-11-10T22:38:33,858 DEBUG [StoreCloser-TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B/ad445e689641457dbfbd04005208429c to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B/ad445e689641457dbfbd04005208429c 2024-11-10T22:38:33,859 DEBUG [StoreCloser-TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B/97e2bd2d217443348f4734caba5f6c63 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B/97e2bd2d217443348f4734caba5f6c63 2024-11-10T22:38:33,860 DEBUG [StoreCloser-TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B/33835994db69410ea57ea2a9ce009173 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B/33835994db69410ea57ea2a9ce009173 2024-11-10T22:38:33,861 DEBUG [StoreCloser-TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B/79f0d3719e57470d8fb01c738594f277 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B/79f0d3719e57470d8fb01c738594f277 2024-11-10T22:38:33,862 DEBUG [StoreCloser-TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B/db2538418a264226914ded5758134157 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B/db2538418a264226914ded5758134157 2024-11-10T22:38:33,864 DEBUG [StoreCloser-TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B/c47796a3673f4344bbee360f081a9433 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B/c47796a3673f4344bbee360f081a9433 2024-11-10T22:38:33,865 DEBUG [StoreCloser-TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B/83e5a514e8b04664aa895cb8f7c55ff3 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B/83e5a514e8b04664aa895cb8f7c55ff3 2024-11-10T22:38:33,866 DEBUG [StoreCloser-TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B/486fc2c33da9494f8fc3be85f64d8c5c to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B/486fc2c33da9494f8fc3be85f64d8c5c 2024-11-10T22:38:33,867 DEBUG [StoreCloser-TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B/d38e66981d7c4aba9163521b3bc9196b to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B/d38e66981d7c4aba9163521b3bc9196b 2024-11-10T22:38:33,868 DEBUG [StoreCloser-TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B/561a785c66bc46d795bd14b7071fd4ab to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B/561a785c66bc46d795bd14b7071fd4ab 2024-11-10T22:38:33,869 DEBUG [StoreCloser-TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B/34549fb4911140f0bbd6df8c7ea97cc0 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B/34549fb4911140f0bbd6df8c7ea97cc0 2024-11-10T22:38:33,870 DEBUG [StoreCloser-TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B/d7c0f2f7d5e541a5a942cd011800c1bb to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B/d7c0f2f7d5e541a5a942cd011800c1bb 2024-11-10T22:38:33,871 DEBUG [StoreCloser-TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B/6176c8a97b074e45aeb42701dd1e9ec4 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B/6176c8a97b074e45aeb42701dd1e9ec4 2024-11-10T22:38:33,872 DEBUG [StoreCloser-TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B/d01eb6c178c64462b69c963b493841bb to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B/d01eb6c178c64462b69c963b493841bb 2024-11-10T22:38:33,890 DEBUG [StoreCloser-TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B/d8609fed597b41d4bd27515f92539596 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B/d8609fed597b41d4bd27515f92539596 2024-11-10T22:38:33,891 DEBUG [StoreCloser-TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B/e34ba7aebece45968f384f5a064604fc to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B/e34ba7aebece45968f384f5a064604fc 2024-11-10T22:38:33,892 DEBUG [StoreCloser-TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B/08ce541f4a6840e5bfb9a2d8ff77bdaf to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B/08ce541f4a6840e5bfb9a2d8ff77bdaf 2024-11-10T22:38:33,893 DEBUG [StoreCloser-TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B/117a94cb724d40808ca895e4e3d80ffa to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B/117a94cb724d40808ca895e4e3d80ffa 2024-11-10T22:38:33,894 DEBUG [StoreCloser-TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B/b67e9ac5e9fd4995a8ad930ee9082032 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B/b67e9ac5e9fd4995a8ad930ee9082032 2024-11-10T22:38:33,895 DEBUG [StoreCloser-TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B/21608d1a1e6e484699ee3b75ebbe7da2 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B/21608d1a1e6e484699ee3b75ebbe7da2 2024-11-10T22:38:33,896 DEBUG [StoreCloser-TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B/675f97d30adc4ae7827e9deed06ed49b to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B/675f97d30adc4ae7827e9deed06ed49b 2024-11-10T22:38:33,897 DEBUG [StoreCloser-TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B/95e1dcb92203407f8d6728e0a413675a to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B/95e1dcb92203407f8d6728e0a413675a 2024-11-10T22:38:33,898 DEBUG [StoreCloser-TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B/c22add54e2b34fd2b2dd30a0b6a8e187 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B/c22add54e2b34fd2b2dd30a0b6a8e187 2024-11-10T22:38:33,899 DEBUG [StoreCloser-TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B/9c453f92666740eeba459d59cfc085d6 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B/9c453f92666740eeba459d59cfc085d6 2024-11-10T22:38:33,900 DEBUG [StoreCloser-TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B/a984f7f7264d48de87eec493b356ebaf to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B/a984f7f7264d48de87eec493b356ebaf 2024-11-10T22:38:33,901 DEBUG [StoreCloser-TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/C/fcfc690455ef47aca9ba62bf8c8e56a0, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/C/fa96840e58464ec8b93c24eeb453d3e8, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/C/c35e8b7199014015a6c53944b35576ec, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/C/65aeccd962204855b671817c73e75948, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/C/92e6e9dd106e48cabd630f7afa571ba4, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/C/252832697c754b54856d36238cdac8f1, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/C/db7c10918d2846f2bd06dd5aab55bd32, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/C/29e0ac2d74a04bfa829bd22869033515, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/C/35bee95a01c640aeb8ac8cd0c20c0b9e, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/C/d27e9e1796f7479cb3919f9aec3b8726, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/C/95e6a53bc8fc48d198df5c48e16ede98, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/C/a8807d8fabb1426b86517d7fbd5027cb, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/C/a5fdb2af495045549dce194f5b74138a, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/C/1e93269280ef4763832d5c3af1f737c3, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/C/eac6ef3171ea4386a4af1c38e482078f, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/C/32e27ec7ee7248ff883dbb33206693ca, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/C/63304d55ae4e4fafaa483faac1592978, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/C/2050541d722e42fd9ae6227e2b1e4400, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/C/cdc99b7a1f794e2899c42d096d5fc1a8, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/C/3321e90762b94aaca2c0818a8ebb56c0, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/C/5d1244c0024545a0a467e8ea1be52b3d, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/C/a1cc561efc054245b49dc4ddaab9b595, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/C/7cd7c715da9944fd9624383ceeac77c0, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/C/2bbd3474af6347c781c4168cc2cebc6b, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/C/e4ed274f3413411984807705284474c2, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/C/b340498e8adf488aa11fb0b5f0f0780c, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/C/589d18958c684845a23a18fd20f7acfe, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/C/ee13695e442744f6b184bdd7bf010c58, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/C/395a96605ccb4a8ca6eeb5eb51ae0c40] to archive 2024-11-10T22:38:33,902 DEBUG [StoreCloser-TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T22:38:33,903 DEBUG [StoreCloser-TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/C/fcfc690455ef47aca9ba62bf8c8e56a0 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/C/fcfc690455ef47aca9ba62bf8c8e56a0 2024-11-10T22:38:33,904 DEBUG [StoreCloser-TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/C/fa96840e58464ec8b93c24eeb453d3e8 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/C/fa96840e58464ec8b93c24eeb453d3e8 2024-11-10T22:38:33,905 DEBUG [StoreCloser-TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/C/c35e8b7199014015a6c53944b35576ec to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/C/c35e8b7199014015a6c53944b35576ec 2024-11-10T22:38:33,906 DEBUG [StoreCloser-TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/C/65aeccd962204855b671817c73e75948 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/C/65aeccd962204855b671817c73e75948 2024-11-10T22:38:33,907 DEBUG [StoreCloser-TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/C/92e6e9dd106e48cabd630f7afa571ba4 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/C/92e6e9dd106e48cabd630f7afa571ba4 2024-11-10T22:38:33,908 DEBUG [StoreCloser-TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/C/252832697c754b54856d36238cdac8f1 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/C/252832697c754b54856d36238cdac8f1 2024-11-10T22:38:33,909 DEBUG [StoreCloser-TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/C/db7c10918d2846f2bd06dd5aab55bd32 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/C/db7c10918d2846f2bd06dd5aab55bd32 2024-11-10T22:38:33,910 DEBUG [StoreCloser-TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/C/29e0ac2d74a04bfa829bd22869033515 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/C/29e0ac2d74a04bfa829bd22869033515 2024-11-10T22:38:33,912 DEBUG [StoreCloser-TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/C/35bee95a01c640aeb8ac8cd0c20c0b9e to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/C/35bee95a01c640aeb8ac8cd0c20c0b9e 2024-11-10T22:38:33,913 DEBUG [StoreCloser-TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/C/d27e9e1796f7479cb3919f9aec3b8726 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/C/d27e9e1796f7479cb3919f9aec3b8726 2024-11-10T22:38:33,914 DEBUG [StoreCloser-TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/C/95e6a53bc8fc48d198df5c48e16ede98 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/C/95e6a53bc8fc48d198df5c48e16ede98 2024-11-10T22:38:33,915 DEBUG [StoreCloser-TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/C/a8807d8fabb1426b86517d7fbd5027cb to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/C/a8807d8fabb1426b86517d7fbd5027cb 2024-11-10T22:38:33,916 DEBUG [StoreCloser-TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/C/a5fdb2af495045549dce194f5b74138a to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/C/a5fdb2af495045549dce194f5b74138a 2024-11-10T22:38:33,917 DEBUG [StoreCloser-TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/C/1e93269280ef4763832d5c3af1f737c3 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/C/1e93269280ef4763832d5c3af1f737c3 2024-11-10T22:38:33,918 DEBUG [StoreCloser-TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/C/eac6ef3171ea4386a4af1c38e482078f to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/C/eac6ef3171ea4386a4af1c38e482078f 2024-11-10T22:38:33,919 DEBUG [StoreCloser-TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/C/32e27ec7ee7248ff883dbb33206693ca to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/C/32e27ec7ee7248ff883dbb33206693ca 2024-11-10T22:38:33,920 DEBUG [StoreCloser-TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/C/63304d55ae4e4fafaa483faac1592978 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/C/63304d55ae4e4fafaa483faac1592978 2024-11-10T22:38:33,922 DEBUG [StoreCloser-TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/C/2050541d722e42fd9ae6227e2b1e4400 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/C/2050541d722e42fd9ae6227e2b1e4400 2024-11-10T22:38:33,923 DEBUG [StoreCloser-TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/C/cdc99b7a1f794e2899c42d096d5fc1a8 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/C/cdc99b7a1f794e2899c42d096d5fc1a8 2024-11-10T22:38:33,924 DEBUG [StoreCloser-TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/C/3321e90762b94aaca2c0818a8ebb56c0 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/C/3321e90762b94aaca2c0818a8ebb56c0 2024-11-10T22:38:33,925 DEBUG [StoreCloser-TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/C/5d1244c0024545a0a467e8ea1be52b3d to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/C/5d1244c0024545a0a467e8ea1be52b3d 2024-11-10T22:38:33,926 DEBUG [StoreCloser-TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/C/a1cc561efc054245b49dc4ddaab9b595 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/C/a1cc561efc054245b49dc4ddaab9b595 2024-11-10T22:38:33,928 DEBUG [StoreCloser-TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/C/7cd7c715da9944fd9624383ceeac77c0 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/C/7cd7c715da9944fd9624383ceeac77c0 2024-11-10T22:38:33,929 DEBUG [StoreCloser-TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/C/2bbd3474af6347c781c4168cc2cebc6b to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/C/2bbd3474af6347c781c4168cc2cebc6b 2024-11-10T22:38:33,930 DEBUG [StoreCloser-TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/C/e4ed274f3413411984807705284474c2 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/C/e4ed274f3413411984807705284474c2 2024-11-10T22:38:33,931 DEBUG [StoreCloser-TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/C/b340498e8adf488aa11fb0b5f0f0780c to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/C/b340498e8adf488aa11fb0b5f0f0780c 2024-11-10T22:38:33,932 DEBUG [StoreCloser-TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/C/589d18958c684845a23a18fd20f7acfe to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/C/589d18958c684845a23a18fd20f7acfe 2024-11-10T22:38:33,934 DEBUG [StoreCloser-TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/C/ee13695e442744f6b184bdd7bf010c58 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/C/ee13695e442744f6b184bdd7bf010c58 2024-11-10T22:38:33,935 DEBUG [StoreCloser-TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/C/395a96605ccb4a8ca6eeb5eb51ae0c40 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/C/395a96605ccb4a8ca6eeb5eb51ae0c40 2024-11-10T22:38:33,940 DEBUG [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/recovered.edits/468.seqid, newMaxSeqId=468, maxSeqId=4 2024-11-10T22:38:33,942 INFO [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. 2024-11-10T22:38:33,942 DEBUG [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1676): Region close journal for d51b53151407a9b9ad82a60ad302f998: Waiting for close lock at 1731278310887Running coprocessor pre-close hooks at 1731278310887Disabling compacts and flushes for region at 1731278310887Disabling writes for close at 1731278312162 (+1275 ms)Obtaining lock to block concurrent updates at 1731278312162Preparing flush snapshotting stores in d51b53151407a9b9ad82a60ad302f998 at 1731278312162Finished memstore snapshotting TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=72720, getOffHeapSize=0, getCellsCount=600 at 1731278312162Flushing stores of TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998. at 1731278312162Flushing d51b53151407a9b9ad82a60ad302f998/A: creating writer at 1731278312162Flushing mob file d51b53151407a9b9ad82a60ad302f998/A: appending metadata at 1731278312167 (+5 ms)Flushing mob file d51b53151407a9b9ad82a60ad302f998/A: closing flushed file at 1731278312167Flushing d51b53151407a9b9ad82a60ad302f998/A: appending metadata at 1731278312575 (+408 ms)Flushing d51b53151407a9b9ad82a60ad302f998/A: closing flushed file at 1731278312575Flushing d51b53151407a9b9ad82a60ad302f998/B: creating writer at 1731278312983 (+408 ms)Flushing d51b53151407a9b9ad82a60ad302f998/B: appending metadata at 1731278312986 (+3 ms)Flushing d51b53151407a9b9ad82a60ad302f998/B: closing flushed file at 1731278312986Flushing d51b53151407a9b9ad82a60ad302f998/C: creating writer at 1731278313394 (+408 ms)Flushing d51b53151407a9b9ad82a60ad302f998/C: appending metadata at 1731278313396 (+2 ms)Flushing d51b53151407a9b9ad82a60ad302f998/C: closing flushed file at 1731278313396Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@349da6ae: reopening flushed file at 1731278313804 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2c87ac58: reopening flushed file at 1731278313809 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@10048f99: reopening flushed file at 1731278313813 (+4 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for d51b53151407a9b9ad82a60ad302f998 in 1655ms, sequenceid=465, compaction requested=true at 1731278313817 (+4 ms)Writing region close event to WAL at 1731278313937 (+120 ms)Running coprocessor post-close hooks at 1731278313941 (+4 ms)Closed at 1731278313942 (+1 ms) 2024-11-10T22:38:33,944 INFO [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] handler.UnassignRegionHandler(157): Closed d51b53151407a9b9ad82a60ad302f998 2024-11-10T22:38:33,944 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=d51b53151407a9b9ad82a60ad302f998, regionState=CLOSED 2024-11-10T22:38:33,946 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=61, ppid=60, state=RUNNABLE, hasLock=false; CloseRegionProcedure d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 because future has completed 2024-11-10T22:38:33,948 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-10T22:38:33,948 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; CloseRegionProcedure d51b53151407a9b9ad82a60ad302f998, server=849d0d71310f,43753,1731278250663 in 3.2120 sec 2024-11-10T22:38:33,950 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=60, resume processing ppid=59 2024-11-10T22:38:33,950 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, ppid=59, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d51b53151407a9b9ad82a60ad302f998, UNASSIGN in 3.2160 sec 2024-11-10T22:38:33,952 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-11-10T22:38:33,953 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 3.2200 sec 2024-11-10T22:38:33,954 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731278313954"}]},"ts":"1731278313954"} 2024-11-10T22:38:33,955 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-10T22:38:33,955 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-10T22:38:33,957 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 3.2300 sec 2024-11-10T22:38:34,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-10T22:38:34,903 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-10T22:38:34,903 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-10T22:38:34,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] procedure2.ProcedureExecutor(1139): Stored pid=62, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T22:38:34,905 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=62, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T22:38:34,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-10T22:38:34,906 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=62, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T22:38:34,908 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998 2024-11-10T22:38:34,910 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A, FileablePath, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B, FileablePath, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/C, FileablePath, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/recovered.edits] 2024-11-10T22:38:34,912 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/220131f58ea94bef8718d1aa9f45fe7b to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/220131f58ea94bef8718d1aa9f45fe7b 2024-11-10T22:38:34,913 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/5d9243e961404207b5c604a350a33733 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/5d9243e961404207b5c604a350a33733 2024-11-10T22:38:34,914 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/6374c379f3844db98663f6fcb766aed4 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/A/6374c379f3844db98663f6fcb766aed4 2024-11-10T22:38:34,916 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B/567d986d313049d3b8017e6d8af309f9 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B/567d986d313049d3b8017e6d8af309f9 2024-11-10T22:38:34,917 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B/7b0044e869ae4b19b1c57aa9f6f82f3e to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B/7b0044e869ae4b19b1c57aa9f6f82f3e 2024-11-10T22:38:34,918 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B/aaaaf6420e944b4ca3ba3b3f030a3346 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/B/aaaaf6420e944b4ca3ba3b3f030a3346 2024-11-10T22:38:34,920 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/C/2d1230aa357a4571a1d54879cb1fad36 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/C/2d1230aa357a4571a1d54879cb1fad36 2024-11-10T22:38:34,921 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/C/53c9754c4bd448f6823c4c2d39712cdb to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/C/53c9754c4bd448f6823c4c2d39712cdb 2024-11-10T22:38:34,922 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/C/9de52c9fbc4447a59d2a0f201f4f4b54 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/C/9de52c9fbc4447a59d2a0f201f4f4b54 2024-11-10T22:38:34,925 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/recovered.edits/468.seqid to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998/recovered.edits/468.seqid 2024-11-10T22:38:34,926 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/d51b53151407a9b9ad82a60ad302f998 2024-11-10T22:38:34,926 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-10T22:38:34,926 DEBUG [PEWorker-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-10T22:38:34,927 DEBUG [PEWorker-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-10T22:38:34,931 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111004d645ea63114b7b80e11a52e482e18d_d51b53151407a9b9ad82a60ad302f998 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111004d645ea63114b7b80e11a52e482e18d_d51b53151407a9b9ad82a60ad302f998 2024-11-10T22:38:34,932 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411100e88f8e80f8143f2a577374cd86795c2_d51b53151407a9b9ad82a60ad302f998 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411100e88f8e80f8143f2a577374cd86795c2_d51b53151407a9b9ad82a60ad302f998 2024-11-10T22:38:34,933 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110120e549b1da84568907d4b41461ce23a_d51b53151407a9b9ad82a60ad302f998 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110120e549b1da84568907d4b41461ce23a_d51b53151407a9b9ad82a60ad302f998 2024-11-10T22:38:34,934 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411101e49ef8c47474d21b1c7623a66f6d645_d51b53151407a9b9ad82a60ad302f998 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411101e49ef8c47474d21b1c7623a66f6d645_d51b53151407a9b9ad82a60ad302f998 2024-11-10T22:38:34,934 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111021d6e25057084a1baaf954c74667b713_d51b53151407a9b9ad82a60ad302f998 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111021d6e25057084a1baaf954c74667b713_d51b53151407a9b9ad82a60ad302f998 2024-11-10T22:38:34,935 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411103d06650a1bcd4d429b21d2a092a72553_d51b53151407a9b9ad82a60ad302f998 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411103d06650a1bcd4d429b21d2a092a72553_d51b53151407a9b9ad82a60ad302f998 2024-11-10T22:38:34,937 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111046de3a749ac34eafb93d85e77c38f4f4_d51b53151407a9b9ad82a60ad302f998 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111046de3a749ac34eafb93d85e77c38f4f4_d51b53151407a9b9ad82a60ad302f998 2024-11-10T22:38:34,938 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411104e16e1c38a194082a44abcec6cfb7a57_d51b53151407a9b9ad82a60ad302f998 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411104e16e1c38a194082a44abcec6cfb7a57_d51b53151407a9b9ad82a60ad302f998 2024-11-10T22:38:34,938 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111064cb5a25a5624021815db288a2c83b65_d51b53151407a9b9ad82a60ad302f998 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111064cb5a25a5624021815db288a2c83b65_d51b53151407a9b9ad82a60ad302f998 2024-11-10T22:38:34,939 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411106df971e0fed34b7f9b003b3de878989c_d51b53151407a9b9ad82a60ad302f998 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411106df971e0fed34b7f9b003b3de878989c_d51b53151407a9b9ad82a60ad302f998 2024-11-10T22:38:34,940 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411106ff5994cbf9a454c90b982bd4f09d554_d51b53151407a9b9ad82a60ad302f998 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411106ff5994cbf9a454c90b982bd4f09d554_d51b53151407a9b9ad82a60ad302f998 2024-11-10T22:38:34,941 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111071cff20015bc4929acb6727ad2c81ccb_d51b53151407a9b9ad82a60ad302f998 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111071cff20015bc4929acb6727ad2c81ccb_d51b53151407a9b9ad82a60ad302f998 2024-11-10T22:38:34,943 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411107a7f38519ecc499eb4c60ce713bf21ac_d51b53151407a9b9ad82a60ad302f998 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411107a7f38519ecc499eb4c60ce713bf21ac_d51b53151407a9b9ad82a60ad302f998 2024-11-10T22:38:34,944 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111083fe874360064eb79e3c7740f4cf200b_d51b53151407a9b9ad82a60ad302f998 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111083fe874360064eb79e3c7740f4cf200b_d51b53151407a9b9ad82a60ad302f998 2024-11-10T22:38:34,945 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111084757fae3322484d891f600a40019542_d51b53151407a9b9ad82a60ad302f998 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111084757fae3322484d891f600a40019542_d51b53151407a9b9ad82a60ad302f998 2024-11-10T22:38:34,946 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411109ede2dc7094f426386b4ade0bd0f7646_d51b53151407a9b9ad82a60ad302f998 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411109ede2dc7094f426386b4ade0bd0f7646_d51b53151407a9b9ad82a60ad302f998 2024-11-10T22:38:34,947 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110b5bf10eb3b28475381e3a597d733d2ac_d51b53151407a9b9ad82a60ad302f998 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110b5bf10eb3b28475381e3a597d733d2ac_d51b53151407a9b9ad82a60ad302f998 2024-11-10T22:38:34,948 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110c0b100caa185402682c0971674d90a98_d51b53151407a9b9ad82a60ad302f998 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110c0b100caa185402682c0971674d90a98_d51b53151407a9b9ad82a60ad302f998 2024-11-10T22:38:34,949 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110c9107bf7237c4b3288d6c7bde8b9b8c3_d51b53151407a9b9ad82a60ad302f998 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110c9107bf7237c4b3288d6c7bde8b9b8c3_d51b53151407a9b9ad82a60ad302f998 2024-11-10T22:38:34,950 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110d7790bd128bf437c90be54014064034b_d51b53151407a9b9ad82a60ad302f998 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110d7790bd128bf437c90be54014064034b_d51b53151407a9b9ad82a60ad302f998 2024-11-10T22:38:34,951 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110dd915cd70f094b6b8901efdfd76cdcc1_d51b53151407a9b9ad82a60ad302f998 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110dd915cd70f094b6b8901efdfd76cdcc1_d51b53151407a9b9ad82a60ad302f998 2024-11-10T22:38:34,952 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110eb9c8c32ba5a4bd4868a3b672eeaad62_d51b53151407a9b9ad82a60ad302f998 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110eb9c8c32ba5a4bd4868a3b672eeaad62_d51b53151407a9b9ad82a60ad302f998 2024-11-10T22:38:34,953 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110fbc328f01ec64b28b5929ac08b6ecae8_d51b53151407a9b9ad82a60ad302f998 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110fbc328f01ec64b28b5929ac08b6ecae8_d51b53151407a9b9ad82a60ad302f998 2024-11-10T22:38:34,954 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110fdc393325c704a8996b5a6e72bd89621_d51b53151407a9b9ad82a60ad302f998 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110fdc393325c704a8996b5a6e72bd89621_d51b53151407a9b9ad82a60ad302f998 2024-11-10T22:38:34,955 DEBUG [PEWorker-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-10T22:38:34,957 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=62, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T22:38:34,960 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-10T22:38:34,962 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-10T22:38:34,963 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=62, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T22:38:34,963 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-10T22:38:34,963 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731278314963"}]},"ts":"9223372036854775807"} 2024-11-10T22:38:34,965 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-10T22:38:34,965 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => d51b53151407a9b9ad82a60ad302f998, NAME => 'TestAcidGuarantees,,1731278286679.d51b53151407a9b9ad82a60ad302f998.', STARTKEY => '', ENDKEY => ''}] 2024-11-10T22:38:34,966 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-10T22:38:34,966 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731278314966"}]},"ts":"9223372036854775807"} 2024-11-10T22:38:34,967 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-10T22:38:34,968 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=62, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T22:38:34,969 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 65 msec 2024-11-10T22:38:35,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-10T22:38:35,013 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-10T22:38:35,013 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-10T22:38:35,022 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobMixedAtomicity Thread=242 (was 237) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-196171798_22 at /127.0.0.1:37132 [Waiting for operation #249] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_2091773998_22 at /127.0.0.1:36894 [Waiting for operation #431] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_2091773998_22 at /127.0.0.1:35426 [Waiting for operation #641] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3340d2ba-2e32-7278-81af-9687863d92cc/cluster_f1d987d6-b0d7-db14-6c2b-f9cd24bc1de9/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3340d2ba-2e32-7278-81af-9687863d92cc/cluster_f1d987d6-b0d7-db14-6c2b-f9cd24bc1de9/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-196171798_22 at /127.0.0.1:44096 [Waiting for operation #642] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=463 (was 455) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=222 (was 161) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=7074 (was 7205) 2024-11-10T22:38:35,031 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testGetAtomicity Thread=242, OpenFileDescriptor=463, MaxFileDescriptor=1048576, SystemLoadAverage=222, ProcessCount=11, AvailableMemoryMB=7074 2024-11-10T22:38:35,032 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-10T22:38:35,032 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-10T22:38:35,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] procedure2.ProcedureExecutor(1139): Stored pid=63, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-10T22:38:35,034 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-10T22:38:35,035 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:35,035 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 63 2024-11-10T22:38:35,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-11-10T22:38:35,035 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-10T22:38:35,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742094_1270 (size=959) 2024-11-10T22:38:35,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-11-10T22:38:35,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-11-10T22:38:35,443 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 665ebb4a1f83b8f4b9dc2b71f1eaf80b, NAME => 'TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39 2024-11-10T22:38:35,448 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742095_1271 (size=53) 2024-11-10T22:38:35,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-11-10T22:38:35,849 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T22:38:35,849 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 665ebb4a1f83b8f4b9dc2b71f1eaf80b, disabling compactions & flushes 2024-11-10T22:38:35,849 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b. 2024-11-10T22:38:35,849 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b. 2024-11-10T22:38:35,849 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b. after waiting 0 ms 2024-11-10T22:38:35,849 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b. 2024-11-10T22:38:35,849 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b. 2024-11-10T22:38:35,849 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 665ebb4a1f83b8f4b9dc2b71f1eaf80b: Waiting for close lock at 1731278315849Disabling compacts and flushes for region at 1731278315849Disabling writes for close at 1731278315849Writing region close event to WAL at 1731278315849Closed at 1731278315849 2024-11-10T22:38:35,850 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-10T22:38:35,850 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731278315850"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731278315850"}]},"ts":"1731278315850"} 2024-11-10T22:38:35,852 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-10T22:38:35,852 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-10T22:38:35,852 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731278315852"}]},"ts":"1731278315852"} 2024-11-10T22:38:35,853 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-10T22:38:35,854 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=665ebb4a1f83b8f4b9dc2b71f1eaf80b, ASSIGN}] 2024-11-10T22:38:35,855 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=665ebb4a1f83b8f4b9dc2b71f1eaf80b, ASSIGN 2024-11-10T22:38:35,855 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=665ebb4a1f83b8f4b9dc2b71f1eaf80b, ASSIGN; state=OFFLINE, location=849d0d71310f,43753,1731278250663; forceNewPlan=false, retain=false 2024-11-10T22:38:36,006 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=64 updating hbase:meta row=665ebb4a1f83b8f4b9dc2b71f1eaf80b, regionState=OPENING, regionLocation=849d0d71310f,43753,1731278250663 2024-11-10T22:38:36,007 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=665ebb4a1f83b8f4b9dc2b71f1eaf80b, ASSIGN because future has completed 2024-11-10T22:38:36,008 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663}] 2024-11-10T22:38:36,162 INFO [RS_OPEN_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b. 2024-11-10T22:38:36,163 DEBUG [RS_OPEN_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(7752): Opening region: {ENCODED => 665ebb4a1f83b8f4b9dc2b71f1eaf80b, NAME => 'TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b.', STARTKEY => '', ENDKEY => ''} 2024-11-10T22:38:36,163 DEBUG [RS_OPEN_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 665ebb4a1f83b8f4b9dc2b71f1eaf80b 2024-11-10T22:38:36,163 DEBUG [RS_OPEN_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T22:38:36,163 DEBUG [RS_OPEN_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(7794): checking encryption for 665ebb4a1f83b8f4b9dc2b71f1eaf80b 2024-11-10T22:38:36,163 DEBUG [RS_OPEN_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(7797): checking classloading for 665ebb4a1f83b8f4b9dc2b71f1eaf80b 2024-11-10T22:38:36,164 INFO [StoreOpener-665ebb4a1f83b8f4b9dc2b71f1eaf80b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 665ebb4a1f83b8f4b9dc2b71f1eaf80b 2024-11-10T22:38:36,166 INFO [StoreOpener-665ebb4a1f83b8f4b9dc2b71f1eaf80b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 665ebb4a1f83b8f4b9dc2b71f1eaf80b columnFamilyName A 2024-11-10T22:38:36,166 DEBUG [StoreOpener-665ebb4a1f83b8f4b9dc2b71f1eaf80b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:36,166 INFO [StoreOpener-665ebb4a1f83b8f4b9dc2b71f1eaf80b-1 {}] regionserver.HStore(327): Store=665ebb4a1f83b8f4b9dc2b71f1eaf80b/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T22:38:36,166 INFO [StoreOpener-665ebb4a1f83b8f4b9dc2b71f1eaf80b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 665ebb4a1f83b8f4b9dc2b71f1eaf80b 2024-11-10T22:38:36,167 INFO [StoreOpener-665ebb4a1f83b8f4b9dc2b71f1eaf80b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 665ebb4a1f83b8f4b9dc2b71f1eaf80b columnFamilyName B 2024-11-10T22:38:36,167 DEBUG [StoreOpener-665ebb4a1f83b8f4b9dc2b71f1eaf80b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:36,168 INFO [StoreOpener-665ebb4a1f83b8f4b9dc2b71f1eaf80b-1 {}] regionserver.HStore(327): Store=665ebb4a1f83b8f4b9dc2b71f1eaf80b/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T22:38:36,168 INFO [StoreOpener-665ebb4a1f83b8f4b9dc2b71f1eaf80b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 665ebb4a1f83b8f4b9dc2b71f1eaf80b 2024-11-10T22:38:36,168 INFO [StoreOpener-665ebb4a1f83b8f4b9dc2b71f1eaf80b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 665ebb4a1f83b8f4b9dc2b71f1eaf80b columnFamilyName C 2024-11-10T22:38:36,169 DEBUG [StoreOpener-665ebb4a1f83b8f4b9dc2b71f1eaf80b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:38:36,169 INFO [StoreOpener-665ebb4a1f83b8f4b9dc2b71f1eaf80b-1 {}] regionserver.HStore(327): Store=665ebb4a1f83b8f4b9dc2b71f1eaf80b/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T22:38:36,169 DEBUG [RS_OPEN_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1038): replaying wal for 665ebb4a1f83b8f4b9dc2b71f1eaf80b 2024-11-10T22:38:36,170 DEBUG [RS_OPEN_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b 2024-11-10T22:38:36,170 DEBUG [RS_OPEN_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b 2024-11-10T22:38:36,171 DEBUG [RS_OPEN_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1048): stopping wal replay for 665ebb4a1f83b8f4b9dc2b71f1eaf80b 2024-11-10T22:38:36,171 DEBUG [RS_OPEN_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1060): Cleaning up temporary data for 665ebb4a1f83b8f4b9dc2b71f1eaf80b 2024-11-10T22:38:36,171 DEBUG [RS_OPEN_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-10T22:38:36,172 DEBUG [RS_OPEN_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1093): writing seq id for 665ebb4a1f83b8f4b9dc2b71f1eaf80b 2024-11-10T22:38:36,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-11-10T22:38:36,174 DEBUG [RS_OPEN_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-10T22:38:36,174 INFO [RS_OPEN_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1114): Opened 665ebb4a1f83b8f4b9dc2b71f1eaf80b; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69484970, jitterRate=0.03540673851966858}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-10T22:38:36,174 DEBUG [RS_OPEN_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 665ebb4a1f83b8f4b9dc2b71f1eaf80b 2024-11-10T22:38:36,175 DEBUG [RS_OPEN_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1006): Region open journal for 665ebb4a1f83b8f4b9dc2b71f1eaf80b: Running coprocessor pre-open hook at 1731278316163Writing region info on filesystem at 1731278316163Initializing all the Stores at 1731278316164 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731278316164Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731278316164Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731278316164Cleaning up temporary data from old regions at 1731278316171 (+7 ms)Running coprocessor post-open hooks at 1731278316174 (+3 ms)Region opened successfully at 1731278316175 (+1 ms) 2024-11-10T22:38:36,175 INFO [RS_OPEN_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., pid=65, masterSystemTime=1731278316159 2024-11-10T22:38:36,177 DEBUG [RS_OPEN_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b. 2024-11-10T22:38:36,177 INFO [RS_OPEN_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b. 2024-11-10T22:38:36,177 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=64 updating hbase:meta row=665ebb4a1f83b8f4b9dc2b71f1eaf80b, regionState=OPEN, openSeqNum=2, regionLocation=849d0d71310f,43753,1731278250663 2024-11-10T22:38:36,179 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=65, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 because future has completed 2024-11-10T22:38:36,181 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-11-10T22:38:36,181 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; OpenRegionProcedure 665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 in 172 msec 2024-11-10T22:38:36,182 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=64, resume processing ppid=63 2024-11-10T22:38:36,183 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, ppid=63, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=665ebb4a1f83b8f4b9dc2b71f1eaf80b, ASSIGN in 327 msec 2024-11-10T22:38:36,183 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-10T22:38:36,183 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731278316183"}]},"ts":"1731278316183"} 2024-11-10T22:38:36,184 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-10T22:38:36,185 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-10T22:38:36,186 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1520 sec 2024-11-10T22:38:37,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-11-10T22:38:37,183 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-10T22:38:37,184 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3fe11fca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:38:37,184 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849d0d71310f,43351,-1 for getting cluster id 2024-11-10T22:38:37,184 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T22:38:37,185 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59ccc382-681c-4202-a6b6-e337b1afc6c7' 2024-11-10T22:38:37,185 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T22:38:37,185 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59ccc382-681c-4202-a6b6-e337b1afc6c7" 2024-11-10T22:38:37,185 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46ac057f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:38:37,185 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849d0d71310f,43351,-1] 2024-11-10T22:38:37,186 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T22:38:37,186 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:38:37,186 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52396, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T22:38:37,187 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@570c82d0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:38:37,187 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T22:38:37,188 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849d0d71310f,43753,1731278250663, seqNum=-1] 2024-11-10T22:38:37,188 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T22:38:37,189 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46558, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T22:38:37,190 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-10T22:38:37,191 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is 849d0d71310f,43351,1731278249979 2024-11-10T22:38:37,191 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@2d1f0354 2024-11-10T22:38:37,191 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-10T22:38:37,192 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52410, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-10T22:38:37,193 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34b5b273, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:38:37,193 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849d0d71310f,43351,-1 for getting cluster id 2024-11-10T22:38:37,193 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T22:38:37,193 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59ccc382-681c-4202-a6b6-e337b1afc6c7' 2024-11-10T22:38:37,193 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T22:38:37,193 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59ccc382-681c-4202-a6b6-e337b1afc6c7" 2024-11-10T22:38:37,194 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53d0548f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:38:37,194 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849d0d71310f,43351,-1] 2024-11-10T22:38:37,194 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T22:38:37,194 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:38:37,195 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52432, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T22:38:37,195 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d79ef2c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:38:37,196 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1240be3c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:38:37,196 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849d0d71310f,43351,-1 for getting cluster id 2024-11-10T22:38:37,196 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T22:38:37,196 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59ccc382-681c-4202-a6b6-e337b1afc6c7' 2024-11-10T22:38:37,196 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T22:38:37,196 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59ccc382-681c-4202-a6b6-e337b1afc6c7" 2024-11-10T22:38:37,197 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2dd624e6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:38:37,197 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849d0d71310f,43351,-1] 2024-11-10T22:38:37,197 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T22:38:37,197 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:38:37,197 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52456, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T22:38:37,198 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f1c7b85, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:38:37,198 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2885eb44, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:38:37,198 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849d0d71310f,43351,-1 for getting cluster id 2024-11-10T22:38:37,198 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T22:38:37,199 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59ccc382-681c-4202-a6b6-e337b1afc6c7' 2024-11-10T22:38:37,199 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T22:38:37,199 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59ccc382-681c-4202-a6b6-e337b1afc6c7" 2024-11-10T22:38:37,199 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@90c88f1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:38:37,199 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849d0d71310f,43351,-1] 2024-11-10T22:38:37,199 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T22:38:37,199 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:38:37,200 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52478, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T22:38:37,200 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52c57189, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:38:37,201 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@692a6227, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:38:37,201 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849d0d71310f,43351,-1 for getting cluster id 2024-11-10T22:38:37,201 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T22:38:37,201 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59ccc382-681c-4202-a6b6-e337b1afc6c7' 2024-11-10T22:38:37,202 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T22:38:37,202 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59ccc382-681c-4202-a6b6-e337b1afc6c7" 2024-11-10T22:38:37,202 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ee199fb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:38:37,202 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849d0d71310f,43351,-1] 2024-11-10T22:38:37,202 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T22:38:37,202 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:38:37,203 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52484, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T22:38:37,203 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f774003, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:38:37,204 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@128d97bc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:38:37,204 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849d0d71310f,43351,-1 for getting cluster id 2024-11-10T22:38:37,204 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T22:38:37,205 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59ccc382-681c-4202-a6b6-e337b1afc6c7' 2024-11-10T22:38:37,205 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T22:38:37,205 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59ccc382-681c-4202-a6b6-e337b1afc6c7" 2024-11-10T22:38:37,205 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2566d302, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:38:37,205 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849d0d71310f,43351,-1] 2024-11-10T22:38:37,205 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T22:38:37,205 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:38:37,206 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52510, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T22:38:37,206 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9ede668, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:38:37,206 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@626de388, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:38:37,206 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849d0d71310f,43351,-1 for getting cluster id 2024-11-10T22:38:37,207 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T22:38:37,207 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59ccc382-681c-4202-a6b6-e337b1afc6c7' 2024-11-10T22:38:37,207 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T22:38:37,207 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59ccc382-681c-4202-a6b6-e337b1afc6c7" 2024-11-10T22:38:37,207 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11108595, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:38:37,207 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849d0d71310f,43351,-1] 2024-11-10T22:38:37,208 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T22:38:37,208 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:38:37,208 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52532, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T22:38:37,209 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@780e90ef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:38:37,209 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47a5c877, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:38:37,209 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849d0d71310f,43351,-1 for getting cluster id 2024-11-10T22:38:37,209 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T22:38:37,210 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59ccc382-681c-4202-a6b6-e337b1afc6c7' 2024-11-10T22:38:37,210 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T22:38:37,210 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59ccc382-681c-4202-a6b6-e337b1afc6c7" 2024-11-10T22:38:37,210 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68033cf6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:38:37,210 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849d0d71310f,43351,-1] 2024-11-10T22:38:37,210 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T22:38:37,210 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:38:37,211 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52544, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T22:38:37,211 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@576d184e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:38:37,212 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@679f7426, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:38:37,212 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849d0d71310f,43351,-1 for getting cluster id 2024-11-10T22:38:37,212 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T22:38:37,212 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59ccc382-681c-4202-a6b6-e337b1afc6c7' 2024-11-10T22:38:37,213 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T22:38:37,213 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59ccc382-681c-4202-a6b6-e337b1afc6c7" 2024-11-10T22:38:37,213 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b2b25e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:38:37,213 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849d0d71310f,43351,-1] 2024-11-10T22:38:37,213 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T22:38:37,213 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:38:37,214 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52568, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T22:38:37,214 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@585d1f9a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:38:37,215 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b71af22, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:38:37,215 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849d0d71310f,43351,-1 for getting cluster id 2024-11-10T22:38:37,215 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T22:38:37,215 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59ccc382-681c-4202-a6b6-e337b1afc6c7' 2024-11-10T22:38:37,215 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T22:38:37,215 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59ccc382-681c-4202-a6b6-e337b1afc6c7" 2024-11-10T22:38:37,215 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77917cc7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:38:37,215 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849d0d71310f,43351,-1] 2024-11-10T22:38:37,216 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T22:38:37,216 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:38:37,216 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52578, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T22:38:37,217 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2897abae, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:38:37,217 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@728d338, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:38:37,217 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849d0d71310f,43351,-1 for getting cluster id 2024-11-10T22:38:37,217 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T22:38:37,218 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59ccc382-681c-4202-a6b6-e337b1afc6c7' 2024-11-10T22:38:37,218 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T22:38:37,218 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59ccc382-681c-4202-a6b6-e337b1afc6c7" 2024-11-10T22:38:37,218 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@453baf7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:38:37,218 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849d0d71310f,43351,-1] 2024-11-10T22:38:37,218 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T22:38:37,218 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:38:37,219 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52608, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T22:38:37,219 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39b9bfca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:38:37,220 DEBUG [Thread-1248 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T22:38:37,220 DEBUG [Thread-1257 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T22:38:37,220 DEBUG [Thread-1255 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T22:38:37,220 DEBUG [Thread-1261 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T22:38:37,220 DEBUG [Thread-1244 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T22:38:37,220 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T22:38:37,221 DEBUG [Thread-1263 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T22:38:37,221 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849d0d71310f,43753,1731278250663, seqNum=-1] 2024-11-10T22:38:37,221 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849d0d71310f,43753,1731278250663, seqNum=-1] 2024-11-10T22:38:37,221 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T22:38:37,221 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T22:38:37,221 DEBUG [Thread-1250 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T22:38:37,221 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849d0d71310f,43753,1731278250663, seqNum=-1] 2024-11-10T22:38:37,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2024-11-10T22:38:37,222 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T22:38:37,222 DEBUG [Thread-1259 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T22:38:37,222 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849d0d71310f,43753,1731278250663, seqNum=-1] 2024-11-10T22:38:37,222 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849d0d71310f,43753,1731278250663, seqNum=-1] 2024-11-10T22:38:37,222 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849d0d71310f,43753,1731278250663, seqNum=-1] 2024-11-10T22:38:37,222 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T22:38:37,222 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T22:38:37,222 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T22:38:37,222 DEBUG [Thread-1246 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T22:38:37,222 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46568, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T22:38:37,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-10T22:38:37,223 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849d0d71310f,43753,1731278250663, seqNum=-1] 2024-11-10T22:38:37,223 DEBUG [Thread-1259 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849d0d71310f,43753,1731278250663, seqNum=-1] 2024-11-10T22:38:37,223 DEBUG [Thread-1259 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T22:38:37,223 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T22:38:37,223 DEBUG [Thread-1252 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T22:38:37,223 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T22:38:37,223 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849d0d71310f,43753,1731278250663, seqNum=-1] 2024-11-10T22:38:37,224 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T22:38:37,224 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46582, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T22:38:37,224 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46606, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T22:38:37,225 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46616, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T22:38:37,225 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T22:38:37,225 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46594, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T22:38:37,225 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T22:38:37,225 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849d0d71310f,43753,1731278250663, seqNum=-1] 2024-11-10T22:38:37,225 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46630, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T22:38:37,225 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2] 2024-11-10T22:38:37,225 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46632, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T22:38:37,225 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T22:38:37,225 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46620, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T22:38:37,225 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46638, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T22:38:37,226 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2] 2024-11-10T22:38:37,227 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2] 2024-11-10T22:38:37,228 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2] 2024-11-10T22:38:37,229 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46654, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T22:38:37,229 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2] 2024-11-10T22:38:37,229 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2] 2024-11-10T22:38:37,229 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2] 2024-11-10T22:38:37,229 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2] 2024-11-10T22:38:37,230 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2] 2024-11-10T22:38:37,231 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2] 2024-11-10T22:38:37,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(8855): Flush requested on 665ebb4a1f83b8f4b9dc2b71f1eaf80b 2024-11-10T22:38:37,240 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T22:38:37,240 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 665ebb4a1f83b8f4b9dc2b71f1eaf80b 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-10T22:38:37,265 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/A/44b73e20feca4c3ca239a03cc2a1c159 is 50, key is test_row_0/A:col10/1731278317234/Put/seqid=0 2024-11-10T22:38:37,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742096_1272 (size=14341) 2024-11-10T22:38:37,297 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:37,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46638 deadline: 1731278377264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:37,298 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:37,298 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:37,298 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:37,302 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:37,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46654 deadline: 1731278377296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:37,303 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:37,303 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:37,303 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:37,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46594 deadline: 1731278377296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:37,303 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:37,303 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:37,303 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:37,303 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:37,303 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:37,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46606 deadline: 1731278377298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:37,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1731278377297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:37,304 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:37,304 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:37,304 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:37,304 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:37,304 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:37,304 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:37,305 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:37,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-10T22:38:37,381 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-10T22:38:37,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b. 2024-11-10T22:38:37,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b. as already flushing 2024-11-10T22:38:37,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b. 2024-11-10T22:38:37,382 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 665ebb4a1f83b8f4b9dc2b71f1eaf80b, NAME => 'TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:37,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 665ebb4a1f83b8f4b9dc2b71f1eaf80b, NAME => 'TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:37,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 665ebb4a1f83b8f4b9dc2b71f1eaf80b, NAME => 'TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 665ebb4a1f83b8f4b9dc2b71f1eaf80b, NAME => 'TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:37,405 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:37,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46638 deadline: 1731278377402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:37,406 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:37,406 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:37,406 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:37,415 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:37,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46654 deadline: 1731278377413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:37,415 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:37,415 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:37,416 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:37,416 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:37,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46594 deadline: 1731278377413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:37,416 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:37,416 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:37,416 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:37,416 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:37,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1731278377413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:37,416 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:37,416 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:37,416 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:37,416 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:37,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46606 deadline: 1731278377414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:37,417 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:37,417 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:37,417 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:37,535 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-10T22:38:37,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b. 2024-11-10T22:38:37,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b. as already flushing 2024-11-10T22:38:37,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b. 2024-11-10T22:38:37,535 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 665ebb4a1f83b8f4b9dc2b71f1eaf80b, NAME => 'TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:37,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 665ebb4a1f83b8f4b9dc2b71f1eaf80b, NAME => 'TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:37,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 665ebb4a1f83b8f4b9dc2b71f1eaf80b, NAME => 'TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 665ebb4a1f83b8f4b9dc2b71f1eaf80b, NAME => 'TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:37,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-10T22:38:37,616 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:37,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46638 deadline: 1731278377613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:37,616 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:37,616 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:37,616 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:37,625 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:37,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46654 deadline: 1731278377622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:37,625 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:37,625 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:37,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46594 deadline: 1731278377622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:37,625 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:37,625 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:37,625 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:37,625 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:37,626 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:37,626 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:37,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46606 deadline: 1731278377622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:37,626 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:37,626 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:37,626 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:37,626 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:37,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1731278377622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:37,626 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:37,626 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:37,626 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:37,678 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/A/44b73e20feca4c3ca239a03cc2a1c159 2024-11-10T22:38:37,689 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-10T22:38:37,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b. 2024-11-10T22:38:37,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b. as already flushing 2024-11-10T22:38:37,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b. 2024-11-10T22:38:37,689 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 665ebb4a1f83b8f4b9dc2b71f1eaf80b, NAME => 'TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:37,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 665ebb4a1f83b8f4b9dc2b71f1eaf80b, NAME => 'TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:37,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 665ebb4a1f83b8f4b9dc2b71f1eaf80b, NAME => 'TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 665ebb4a1f83b8f4b9dc2b71f1eaf80b, NAME => 'TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:37,709 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/B/807c2b356f13449ba86b4a0760000186 is 50, key is test_row_0/B:col10/1731278317234/Put/seqid=0 2024-11-10T22:38:37,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742097_1273 (size=12001) 2024-11-10T22:38:37,715 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/B/807c2b356f13449ba86b4a0760000186 2024-11-10T22:38:37,741 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/C/7ecf0e35f2a94aafb73b2a9b2311a14c is 50, key is test_row_0/C:col10/1731278317234/Put/seqid=0 2024-11-10T22:38:37,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742098_1274 (size=12001) 2024-11-10T22:38:37,746 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/C/7ecf0e35f2a94aafb73b2a9b2311a14c 2024-11-10T22:38:37,752 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/A/44b73e20feca4c3ca239a03cc2a1c159 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/A/44b73e20feca4c3ca239a03cc2a1c159 2024-11-10T22:38:37,756 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/A/44b73e20feca4c3ca239a03cc2a1c159, entries=200, sequenceid=16, filesize=14.0 K 2024-11-10T22:38:37,757 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/B/807c2b356f13449ba86b4a0760000186 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/B/807c2b356f13449ba86b4a0760000186 2024-11-10T22:38:37,761 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/B/807c2b356f13449ba86b4a0760000186, entries=150, sequenceid=16, filesize=11.7 K 2024-11-10T22:38:37,763 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/C/7ecf0e35f2a94aafb73b2a9b2311a14c as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/C/7ecf0e35f2a94aafb73b2a9b2311a14c 2024-11-10T22:38:37,768 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/C/7ecf0e35f2a94aafb73b2a9b2311a14c, entries=150, sequenceid=16, filesize=11.7 K 2024-11-10T22:38:37,769 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 665ebb4a1f83b8f4b9dc2b71f1eaf80b in 529ms, sequenceid=16, compaction requested=false 2024-11-10T22:38:37,769 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 665ebb4a1f83b8f4b9dc2b71f1eaf80b: 2024-11-10T22:38:37,843 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-10T22:38:37,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b. 2024-11-10T22:38:37,843 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2902): Flushing 665ebb4a1f83b8f4b9dc2b71f1eaf80b 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-10T22:38:37,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/A/94ce0a91e9654a14aed156dbdadba18a is 50, key is test_row_0/A:col10/1731278317268/Put/seqid=0 2024-11-10T22:38:37,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-10T22:38:37,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742099_1275 (size=12001) 2024-11-10T22:38:37,859 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/A/94ce0a91e9654a14aed156dbdadba18a 2024-11-10T22:38:37,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/B/9271e946e0a94f538c8978b87e21ba61 is 50, key is test_row_0/B:col10/1731278317268/Put/seqid=0 2024-11-10T22:38:37,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742100_1276 (size=12001) 2024-11-10T22:38:37,878 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/B/9271e946e0a94f538c8978b87e21ba61 2024-11-10T22:38:37,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/C/9493540e1fb2421d9402b0c6a988fe04 is 50, key is test_row_0/C:col10/1731278317268/Put/seqid=0 2024-11-10T22:38:37,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742101_1277 (size=12001) 2024-11-10T22:38:37,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(8855): Flush requested on 665ebb4a1f83b8f4b9dc2b71f1eaf80b 2024-11-10T22:38:37,925 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b. as already flushing 2024-11-10T22:38:37,950 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:37,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1731278377944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:37,950 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:37,950 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:37,950 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:37,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46594 deadline: 1731278377944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:37,950 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:37,950 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:37,950 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:37,951 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:37,951 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:37,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46606 deadline: 1731278377945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:37,951 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:37,951 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:37,951 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:37,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46638 deadline: 1731278377945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:37,951 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:37,951 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:37,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46654 deadline: 1731278377946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:37,951 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:37,951 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:37,951 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:37,951 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:37,951 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:37,951 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:38,054 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:38,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1731278378052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:38,055 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:38,055 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:38,055 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:38,055 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:38,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46594 deadline: 1731278378052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:38,055 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:38,055 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:38,055 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:38,065 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:38,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46606 deadline: 1731278378062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:38,065 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:38,066 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:38,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46638 deadline: 1731278378062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:38,066 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:38,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46654 deadline: 1731278378062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:38,066 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:38,066 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:38,066 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:38,066 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:38,066 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:38,066 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:38,066 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:38,066 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:38,264 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:38,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1731278378262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:38,265 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:38,265 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:38,265 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:38,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46594 deadline: 1731278378262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:38,265 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:38,265 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:38,265 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:38,265 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:38,275 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:38,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46606 deadline: 1731278378272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:38,275 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:38,275 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:38,276 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:38,276 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:38,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46638 deadline: 1731278378272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:38,276 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:38,276 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:38,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46654 deadline: 1731278378272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:38,276 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:38,276 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:38,276 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:38,276 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:38,276 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:38,310 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/C/9493540e1fb2421d9402b0c6a988fe04 2024-11-10T22:38:38,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/A/94ce0a91e9654a14aed156dbdadba18a as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/A/94ce0a91e9654a14aed156dbdadba18a 2024-11-10T22:38:38,319 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/A/94ce0a91e9654a14aed156dbdadba18a, entries=150, sequenceid=37, filesize=11.7 K 2024-11-10T22:38:38,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/B/9271e946e0a94f538c8978b87e21ba61 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/B/9271e946e0a94f538c8978b87e21ba61 2024-11-10T22:38:38,324 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/B/9271e946e0a94f538c8978b87e21ba61, entries=150, sequenceid=37, filesize=11.7 K 2024-11-10T22:38:38,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/C/9493540e1fb2421d9402b0c6a988fe04 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/C/9493540e1fb2421d9402b0c6a988fe04 2024-11-10T22:38:38,330 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/C/9493540e1fb2421d9402b0c6a988fe04, entries=150, sequenceid=37, filesize=11.7 K 2024-11-10T22:38:38,331 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=93.93 KB/96180 for 665ebb4a1f83b8f4b9dc2b71f1eaf80b in 488ms, sequenceid=37, compaction requested=false 2024-11-10T22:38:38,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2603): Flush status journal for 665ebb4a1f83b8f4b9dc2b71f1eaf80b: 2024-11-10T22:38:38,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b. 2024-11-10T22:38:38,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=67 2024-11-10T22:38:38,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.HMaster(4169): Remote procedure done, pid=67 2024-11-10T22:38:38,335 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-11-10T22:38:38,335 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1080 sec 2024-11-10T22:38:38,337 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 1.1150 sec 2024-11-10T22:38:38,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-10T22:38:38,362 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T22:38:38,363 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T22:38:38,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-11-10T22:38:38,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-10T22:38:38,365 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T22:38:38,365 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T22:38:38,365 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T22:38:38,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-10T22:38:38,518 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-10T22:38:38,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b. 2024-11-10T22:38:38,518 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing 665ebb4a1f83b8f4b9dc2b71f1eaf80b 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-10T22:38:38,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/A/50c70f061cee45219629bc9b670e7499 is 50, key is test_row_0/A:col10/1731278317944/Put/seqid=0 2024-11-10T22:38:38,528 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742102_1278 (size=12001) 2024-11-10T22:38:38,529 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=54 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/A/50c70f061cee45219629bc9b670e7499 2024-11-10T22:38:38,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/B/76f5de29caa54e768d4ae8071d5ee438 is 50, key is test_row_0/B:col10/1731278317944/Put/seqid=0 2024-11-10T22:38:38,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742103_1279 (size=12001) 2024-11-10T22:38:38,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(8855): Flush requested on 665ebb4a1f83b8f4b9dc2b71f1eaf80b 2024-11-10T22:38:38,574 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b. as already flushing 2024-11-10T22:38:38,600 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:38,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46606 deadline: 1731278378594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:38,601 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:38,601 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:38,601 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:38,601 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:38,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46638 deadline: 1731278378595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:38,601 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:38,601 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:38,601 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:38,601 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:38,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46654 deadline: 1731278378596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:38,601 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:38,601 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:38,602 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:38,602 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:38,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46594 deadline: 1731278378597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:38,602 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:38,602 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:38,602 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:38,605 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:38,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1731278378600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:38,605 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:38,605 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:38,605 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:38,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-10T22:38:38,716 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:38,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46606 deadline: 1731278378713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:38,716 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:38,716 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:38,717 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:38,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46654 deadline: 1731278378713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:38,717 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:38,717 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:38,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46638 deadline: 1731278378713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:38,717 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:38,717 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:38,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46594 deadline: 1731278378713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:38,717 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:38,717 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:38,717 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:38,717 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:38,717 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:38,717 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:38,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1731278378713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:38,717 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:38,717 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:38,717 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:38,717 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:38,717 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:38,717 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:38,923 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:38,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46606 deadline: 1731278378923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:38,924 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:38,924 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:38,924 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:38,926 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:38,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46654 deadline: 1731278378923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:38,926 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:38,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46638 deadline: 1731278378923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:38,926 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:38,926 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:38,926 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:38,926 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:38,926 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:38,926 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:38,926 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:38,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46594 deadline: 1731278378923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:38,927 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:38,927 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:38,927 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:38,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1731278378923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:38,927 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:38,927 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:38,927 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:38,927 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:38,946 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=54 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/B/76f5de29caa54e768d4ae8071d5ee438 2024-11-10T22:38:38,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/C/63b834ae163b49619c3041236aeeb61b is 50, key is test_row_0/C:col10/1731278317944/Put/seqid=0 2024-11-10T22:38:38,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742104_1280 (size=12001) 2024-11-10T22:38:38,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-10T22:38:39,236 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:39,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46606 deadline: 1731278379233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:39,236 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:39,236 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:39,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46654 deadline: 1731278379233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:39,236 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:39,236 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:39,237 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:39,237 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:39,237 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:39,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46638 deadline: 1731278379233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:39,237 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:39,237 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:39,237 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:39,237 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:39,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46594 deadline: 1731278379233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:39,237 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:39,237 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:39,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1731278379233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:39,237 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:39,237 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:39,237 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:39,238 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:39,238 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:39,238 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:39,369 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=54 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/C/63b834ae163b49619c3041236aeeb61b 2024-11-10T22:38:39,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/A/50c70f061cee45219629bc9b670e7499 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/A/50c70f061cee45219629bc9b670e7499 2024-11-10T22:38:39,378 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/A/50c70f061cee45219629bc9b670e7499, entries=150, sequenceid=54, filesize=11.7 K 2024-11-10T22:38:39,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/B/76f5de29caa54e768d4ae8071d5ee438 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/B/76f5de29caa54e768d4ae8071d5ee438 2024-11-10T22:38:39,383 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/B/76f5de29caa54e768d4ae8071d5ee438, entries=150, sequenceid=54, filesize=11.7 K 2024-11-10T22:38:39,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/C/63b834ae163b49619c3041236aeeb61b as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/C/63b834ae163b49619c3041236aeeb61b 2024-11-10T22:38:39,388 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/C/63b834ae163b49619c3041236aeeb61b, entries=150, sequenceid=54, filesize=11.7 K 2024-11-10T22:38:39,389 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for 665ebb4a1f83b8f4b9dc2b71f1eaf80b in 871ms, sequenceid=54, compaction requested=true 2024-11-10T22:38:39,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for 665ebb4a1f83b8f4b9dc2b71f1eaf80b: 2024-11-10T22:38:39,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b. 2024-11-10T22:38:39,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-11-10T22:38:39,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-11-10T22:38:39,392 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-11-10T22:38:39,392 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0250 sec 2024-11-10T22:38:39,394 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 1.0300 sec 2024-11-10T22:38:39,470 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-10T22:38:39,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-10T22:38:39,503 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T22:38:39,503 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T22:38:39,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-11-10T22:38:39,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-10T22:38:39,505 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T22:38:39,507 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T22:38:39,507 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T22:38:39,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-10T22:38:39,660 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-10T22:38:39,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b. 2024-11-10T22:38:39,660 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing 665ebb4a1f83b8f4b9dc2b71f1eaf80b 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-10T22:38:39,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/A/35b31f57a1ec48ec99d4e1cb8d385709 is 50, key is test_row_0/A:col10/1731278318593/Put/seqid=0 2024-11-10T22:38:39,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742105_1281 (size=12001) 2024-11-10T22:38:39,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(8855): Flush requested on 665ebb4a1f83b8f4b9dc2b71f1eaf80b 2024-11-10T22:38:39,744 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b. as already flushing 2024-11-10T22:38:39,779 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:39,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46654 deadline: 1731278379775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:39,779 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:39,780 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:39,780 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:39,783 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:39,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46638 deadline: 1731278379776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:39,783 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:39,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46594 deadline: 1731278379777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:39,783 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:39,783 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:39,783 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:39,783 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:39,783 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:39,783 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:39,783 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:39,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1731278379778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:39,784 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:39,784 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:39,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46606 deadline: 1731278379779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:39,784 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:39,784 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:39,784 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:39,784 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:39,784 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:39,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-10T22:38:39,886 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:39,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46654 deadline: 1731278379882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:39,887 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:39,887 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:39,887 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:39,896 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:39,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46638 deadline: 1731278379893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:39,896 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:39,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46594 deadline: 1731278379893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:39,897 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:39,897 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:39,897 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:39,897 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:39,897 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:39,897 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:39,897 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:39,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1731278379893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:39,897 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:39,897 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:39,897 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:39,897 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:39,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46606 deadline: 1731278379893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:39,898 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:39,898 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:39,898 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:40,074 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=73 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/A/35b31f57a1ec48ec99d4e1cb8d385709 2024-11-10T22:38:40,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/B/e44438a5288143d1a4d9c6263d4c961f is 50, key is test_row_0/B:col10/1731278318593/Put/seqid=0 2024-11-10T22:38:40,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742106_1282 (size=12001) 2024-11-10T22:38:40,089 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=73 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/B/e44438a5288143d1a4d9c6263d4c961f 2024-11-10T22:38:40,096 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:40,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46654 deadline: 1731278380093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:40,097 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:40,097 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:40,097 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:40,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/C/987e7e90493940eca649e12e8376c494 is 50, key is test_row_0/C:col10/1731278318593/Put/seqid=0 2024-11-10T22:38:40,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742107_1283 (size=12001) 2024-11-10T22:38:40,103 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:40,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46638 deadline: 1731278380103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:40,104 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:40,104 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:40,104 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:40,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46594 deadline: 1731278380103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:40,104 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:40,104 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:40,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1731278380103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:40,104 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:40,104 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:40,104 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:40,104 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:40,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46606 deadline: 1731278380103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:40,105 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:40,105 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:40,105 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:40,105 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:40,105 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:40,105 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:40,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-10T22:38:40,405 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:40,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46654 deadline: 1731278380403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:40,406 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:40,406 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:40,406 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:40,415 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:40,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46638 deadline: 1731278380413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:40,415 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:40,415 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:40,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1731278380413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:40,415 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:40,415 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:40,416 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:40,416 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:40,416 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:40,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46594 deadline: 1731278380413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:40,416 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:40,416 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:40,416 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:40,416 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:40,417 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:40,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46606 deadline: 1731278380413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:40,417 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:40,417 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:40,417 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:40,503 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=73 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/C/987e7e90493940eca649e12e8376c494 2024-11-10T22:38:40,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/A/35b31f57a1ec48ec99d4e1cb8d385709 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/A/35b31f57a1ec48ec99d4e1cb8d385709 2024-11-10T22:38:40,513 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/A/35b31f57a1ec48ec99d4e1cb8d385709, entries=150, sequenceid=73, filesize=11.7 K 2024-11-10T22:38:40,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/B/e44438a5288143d1a4d9c6263d4c961f as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/B/e44438a5288143d1a4d9c6263d4c961f 2024-11-10T22:38:40,519 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/B/e44438a5288143d1a4d9c6263d4c961f, entries=150, sequenceid=73, filesize=11.7 K 2024-11-10T22:38:40,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/C/987e7e90493940eca649e12e8376c494 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/C/987e7e90493940eca649e12e8376c494 2024-11-10T22:38:40,523 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/C/987e7e90493940eca649e12e8376c494, entries=150, sequenceid=73, filesize=11.7 K 2024-11-10T22:38:40,524 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=100.63 KB/103050 for 665ebb4a1f83b8f4b9dc2b71f1eaf80b in 864ms, sequenceid=73, compaction requested=true 2024-11-10T22:38:40,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for 665ebb4a1f83b8f4b9dc2b71f1eaf80b: 2024-11-10T22:38:40,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b. 2024-11-10T22:38:40,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-11-10T22:38:40,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-11-10T22:38:40,528 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-11-10T22:38:40,528 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0190 sec 2024-11-10T22:38:40,529 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 1.0240 sec 2024-11-10T22:38:40,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-10T22:38:40,642 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T22:38:40,643 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T22:38:40,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-11-10T22:38:40,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-10T22:38:40,644 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T22:38:40,645 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T22:38:40,645 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T22:38:40,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-10T22:38:40,798 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-10T22:38:40,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b. 2024-11-10T22:38:40,798 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing 665ebb4a1f83b8f4b9dc2b71f1eaf80b 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-10T22:38:40,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/A/0ecc17aa9d35429b9245e68ba589fdfe is 50, key is test_row_0/A:col10/1731278319777/Put/seqid=0 2024-11-10T22:38:40,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742108_1284 (size=12001) 2024-11-10T22:38:40,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(8855): Flush requested on 665ebb4a1f83b8f4b9dc2b71f1eaf80b 2024-11-10T22:38:40,916 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b. as already flushing 2024-11-10T22:38:40,943 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:40,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1731278380936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:40,943 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:40,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46606 deadline: 1731278380937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:40,943 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:40,943 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:40,943 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:40,943 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:40,944 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:40,944 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:40,944 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:40,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46654 deadline: 1731278380938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:40,944 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:40,944 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:40,944 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:40,944 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:40,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46638 deadline: 1731278380939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:40,944 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:40,944 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:40,944 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:40,945 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:40,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46594 deadline: 1731278380940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:40,946 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:40,946 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:40,946 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:40,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-10T22:38:41,055 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:41,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1731278381053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:41,056 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:41,056 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:41,056 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:41,056 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:41,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46606 deadline: 1731278381053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:41,056 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:41,056 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:41,056 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:41,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46654 deadline: 1731278381053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:41,056 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:41,057 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:41,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46638 deadline: 1731278381053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:41,057 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:41,057 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:41,057 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:41,057 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:41,057 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:41,057 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:41,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46594 deadline: 1731278381053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:41,057 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:41,057 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:41,057 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:41,057 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:41,210 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/A/0ecc17aa9d35429b9245e68ba589fdfe 2024-11-10T22:38:41,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/B/babab62d94414052807fb76475c30695 is 50, key is test_row_0/B:col10/1731278319777/Put/seqid=0 2024-11-10T22:38:41,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742109_1285 (size=12001) 2024-11-10T22:38:41,268 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:41,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1731278381263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:41,268 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:41,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46606 deadline: 1731278381263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:41,269 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:41,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46654 deadline: 1731278381263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:41,269 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:41,269 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:41,269 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:41,269 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:41,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46638 deadline: 1731278381263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:41,269 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:41,269 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:41,269 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:41,269 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:41,269 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:41,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46594 deadline: 1731278381263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:41,269 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:41,269 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:41,269 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:41,269 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:41,269 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:41,269 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:41,269 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:41,269 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:41,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-10T22:38:41,576 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:41,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46606 deadline: 1731278381572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:41,576 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:41,576 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:41,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46654 deadline: 1731278381572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:41,576 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:41,577 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:41,577 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:41,577 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:41,577 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:41,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46638 deadline: 1731278381573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:41,577 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:41,577 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:41,577 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:41,577 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:41,577 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:41,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46594 deadline: 1731278381573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:41,577 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:41,578 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:41,578 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:41,584 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:41,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1731278381582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:41,584 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:41,584 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:41,584 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:41,624 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/B/babab62d94414052807fb76475c30695 2024-11-10T22:38:41,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/C/25a2e22f46064e8dba537d1428553558 is 50, key is test_row_0/C:col10/1731278319777/Put/seqid=0 2024-11-10T22:38:41,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742110_1286 (size=12001) 2024-11-10T22:38:41,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-10T22:38:42,044 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/C/25a2e22f46064e8dba537d1428553558 2024-11-10T22:38:42,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/A/0ecc17aa9d35429b9245e68ba589fdfe as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/A/0ecc17aa9d35429b9245e68ba589fdfe 2024-11-10T22:38:42,054 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/A/0ecc17aa9d35429b9245e68ba589fdfe, entries=150, sequenceid=91, filesize=11.7 K 2024-11-10T22:38:42,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/B/babab62d94414052807fb76475c30695 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/B/babab62d94414052807fb76475c30695 2024-11-10T22:38:42,059 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/B/babab62d94414052807fb76475c30695, entries=150, sequenceid=91, filesize=11.7 K 2024-11-10T22:38:42,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/C/25a2e22f46064e8dba537d1428553558 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/C/25a2e22f46064e8dba537d1428553558 2024-11-10T22:38:42,067 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/C/25a2e22f46064e8dba537d1428553558, entries=150, sequenceid=91, filesize=11.7 K 2024-11-10T22:38:42,068 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for 665ebb4a1f83b8f4b9dc2b71f1eaf80b in 1270ms, sequenceid=91, compaction requested=true 2024-11-10T22:38:42,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for 665ebb4a1f83b8f4b9dc2b71f1eaf80b: 2024-11-10T22:38:42,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b. 2024-11-10T22:38:42,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-11-10T22:38:42,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-11-10T22:38:42,073 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-11-10T22:38:42,073 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4250 sec 2024-11-10T22:38:42,075 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 1.4310 sec 2024-11-10T22:38:42,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(8855): Flush requested on 665ebb4a1f83b8f4b9dc2b71f1eaf80b 2024-11-10T22:38:42,116 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T22:38:42,116 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 665ebb4a1f83b8f4b9dc2b71f1eaf80b 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-10T22:38:42,121 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/A/9486bfc238c741c6a3a0bf754b8fbe10 is 50, key is test_row_0/A:col10/1731278322116/Put/seqid=0 2024-11-10T22:38:42,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742111_1287 (size=14341) 2024-11-10T22:38:42,125 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=113 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/A/9486bfc238c741c6a3a0bf754b8fbe10 2024-11-10T22:38:42,133 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/B/078c225608d14ac0b9582a1ba5698115 is 50, key is test_row_0/B:col10/1731278322116/Put/seqid=0 2024-11-10T22:38:42,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742112_1288 (size=12001) 2024-11-10T22:38:42,137 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=113 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/B/078c225608d14ac0b9582a1ba5698115 2024-11-10T22:38:42,137 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:42,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46606 deadline: 1731278382132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:42,138 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:42,138 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:42,138 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:42,142 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:42,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46638 deadline: 1731278382134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:42,142 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:42,142 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:42,142 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:42,142 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:42,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46654 deadline: 1731278382135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:42,143 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:42,143 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:42,143 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:42,143 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:42,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46594 deadline: 1731278382136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:42,143 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:42,143 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:42,143 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:42,144 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:42,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1731278382138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:42,145 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/C/b312b7db25014a4cbbb36a0cc71bd30d is 50, key is test_row_0/C:col10/1731278322116/Put/seqid=0 2024-11-10T22:38:42,146 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:42,146 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:42,146 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:42,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742113_1289 (size=12001) 2024-11-10T22:38:42,245 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:42,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46606 deadline: 1731278382243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:42,246 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:42,246 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:42,246 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:42,253 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:42,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46654 deadline: 1731278382252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:42,254 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:42,254 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:42,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46638 deadline: 1731278382252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:42,254 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:42,254 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:42,254 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:42,254 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:42,254 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:42,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46594 deadline: 1731278382252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:42,254 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:42,254 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:42,254 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:42,254 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:42,254 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:42,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1731278382252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:42,254 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:42,255 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:42,255 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:42,454 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:42,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46606 deadline: 1731278382453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:42,455 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:42,455 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:42,455 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:42,465 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:42,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46638 deadline: 1731278382462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:42,465 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:42,465 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:42,465 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:42,465 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:42,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46654 deadline: 1731278382462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:42,466 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:42,466 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:42,466 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:42,466 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:42,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46594 deadline: 1731278382462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:42,466 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:42,466 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:42,466 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:42,466 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:42,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1731278382462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:42,466 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:42,466 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:42,466 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:42,550 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=113 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/C/b312b7db25014a4cbbb36a0cc71bd30d 2024-11-10T22:38:42,555 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/A/9486bfc238c741c6a3a0bf754b8fbe10 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/A/9486bfc238c741c6a3a0bf754b8fbe10 2024-11-10T22:38:42,559 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/A/9486bfc238c741c6a3a0bf754b8fbe10, entries=200, sequenceid=113, filesize=14.0 K 2024-11-10T22:38:42,559 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/B/078c225608d14ac0b9582a1ba5698115 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/B/078c225608d14ac0b9582a1ba5698115 2024-11-10T22:38:42,564 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/B/078c225608d14ac0b9582a1ba5698115, entries=150, sequenceid=113, filesize=11.7 K 2024-11-10T22:38:42,565 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/C/b312b7db25014a4cbbb36a0cc71bd30d as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/C/b312b7db25014a4cbbb36a0cc71bd30d 2024-11-10T22:38:42,570 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/C/b312b7db25014a4cbbb36a0cc71bd30d, entries=150, sequenceid=113, filesize=11.7 K 2024-11-10T22:38:42,570 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 665ebb4a1f83b8f4b9dc2b71f1eaf80b in 454ms, sequenceid=113, compaction requested=true 2024-11-10T22:38:42,571 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 665ebb4a1f83b8f4b9dc2b71f1eaf80b: 2024-11-10T22:38:42,571 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 665ebb4a1f83b8f4b9dc2b71f1eaf80b:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T22:38:42,571 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T22:38:42,571 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-11-10T22:38:42,571 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 665ebb4a1f83b8f4b9dc2b71f1eaf80b:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T22:38:42,571 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T22:38:42,571 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 665ebb4a1f83b8f4b9dc2b71f1eaf80b:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T22:38:42,571 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T22:38:42,571 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-11-10T22:38:42,573 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 76686 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-11-10T22:38:42,573 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HStore(1541): 665ebb4a1f83b8f4b9dc2b71f1eaf80b/A is initiating minor compaction (all files) 2024-11-10T22:38:42,573 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 665ebb4a1f83b8f4b9dc2b71f1eaf80b/A in TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b. 2024-11-10T22:38:42,574 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/A/44b73e20feca4c3ca239a03cc2a1c159, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/A/94ce0a91e9654a14aed156dbdadba18a, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/A/50c70f061cee45219629bc9b670e7499, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/A/35b31f57a1ec48ec99d4e1cb8d385709, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/A/0ecc17aa9d35429b9245e68ba589fdfe, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/A/9486bfc238c741c6a3a0bf754b8fbe10] into tmpdir=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp, totalSize=74.9 K 2024-11-10T22:38:42,574 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 72006 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-11-10T22:38:42,574 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1541): 665ebb4a1f83b8f4b9dc2b71f1eaf80b/B is initiating minor compaction (all files) 2024-11-10T22:38:42,574 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 665ebb4a1f83b8f4b9dc2b71f1eaf80b/B in TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b. 2024-11-10T22:38:42,574 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/B/807c2b356f13449ba86b4a0760000186, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/B/9271e946e0a94f538c8978b87e21ba61, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/B/76f5de29caa54e768d4ae8071d5ee438, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/B/e44438a5288143d1a4d9c6263d4c961f, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/B/babab62d94414052807fb76475c30695, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/B/078c225608d14ac0b9582a1ba5698115] into tmpdir=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp, totalSize=70.3 K 2024-11-10T22:38:42,575 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 44b73e20feca4c3ca239a03cc2a1c159, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731278317234 2024-11-10T22:38:42,575 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting 807c2b356f13449ba86b4a0760000186, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731278317234 2024-11-10T22:38:42,575 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 94ce0a91e9654a14aed156dbdadba18a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731278317264 2024-11-10T22:38:42,575 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting 9271e946e0a94f538c8978b87e21ba61, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731278317264 2024-11-10T22:38:42,575 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 50c70f061cee45219629bc9b670e7499, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731278317937 2024-11-10T22:38:42,575 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting 76f5de29caa54e768d4ae8071d5ee438, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731278317937 2024-11-10T22:38:42,576 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting e44438a5288143d1a4d9c6263d4c961f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=73, earliestPutTs=1731278318593 2024-11-10T22:38:42,576 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 35b31f57a1ec48ec99d4e1cb8d385709, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=73, earliestPutTs=1731278318593 2024-11-10T22:38:42,576 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting babab62d94414052807fb76475c30695, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1731278319768 2024-11-10T22:38:42,576 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0ecc17aa9d35429b9245e68ba589fdfe, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1731278319768 2024-11-10T22:38:42,576 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting 078c225608d14ac0b9582a1ba5698115, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1731278322085 2024-11-10T22:38:42,576 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9486bfc238c741c6a3a0bf754b8fbe10, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1731278322085 2024-11-10T22:38:42,588 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 665ebb4a1f83b8f4b9dc2b71f1eaf80b#A#compaction#239 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T22:38:42,588 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/A/e9c98768aac84abb9b1076a9e5518e9b is 50, key is test_row_0/A:col10/1731278322116/Put/seqid=0 2024-11-10T22:38:42,591 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 665ebb4a1f83b8f4b9dc2b71f1eaf80b#B#compaction#240 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T22:38:42,592 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/B/414179b8c00649aea83c5f77dfce8c58 is 50, key is test_row_0/B:col10/1731278322116/Put/seqid=0 2024-11-10T22:38:42,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742114_1290 (size=12207) 2024-11-10T22:38:42,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742115_1291 (size=12207) 2024-11-10T22:38:42,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(8855): Flush requested on 665ebb4a1f83b8f4b9dc2b71f1eaf80b 2024-11-10T22:38:42,766 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T22:38:42,766 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 665ebb4a1f83b8f4b9dc2b71f1eaf80b 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-10T22:38:42,774 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/A/63032a8b961a47c9b5fcecee5480b6a0 is 50, key is test_row_0/A:col10/1731278322765/Put/seqid=0 2024-11-10T22:38:42,780 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742116_1292 (size=11997) 2024-11-10T22:38:42,781 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=128 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/A/63032a8b961a47c9b5fcecee5480b6a0 2024-11-10T22:38:42,788 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/B/b9ef504fcdc74e62a9fc47cc64ec5e0b is 50, key is test_row_0/B:col10/1731278322765/Put/seqid=0 2024-11-10T22:38:42,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742117_1293 (size=9657) 2024-11-10T22:38:42,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-10T22:38:42,803 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T22:38:42,804 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T22:38:42,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-11-10T22:38:42,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-10T22:38:42,807 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T22:38:42,808 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T22:38:42,808 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T22:38:42,814 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:42,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46654 deadline: 1731278382806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:42,815 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:42,815 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:42,815 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:42,816 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:42,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46594 deadline: 1731278382808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:42,817 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:42,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1731278382809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:42,817 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:42,817 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:42,817 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:42,818 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:42,818 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:42,818 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:42,822 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:42,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46606 deadline: 1731278382812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:42,823 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:42,823 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:42,823 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:42,823 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:42,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46638 deadline: 1731278382815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:42,823 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:42,824 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:42,824 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:42,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-10T22:38:42,928 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:42,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46654 deadline: 1731278382923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:42,928 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:42,928 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:42,928 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:42,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46594 deadline: 1731278382923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:42,929 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:42,929 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:42,929 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:42,929 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:42,929 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:42,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1731278382923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:42,929 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:42,929 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:42,929 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:42,937 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:42,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46606 deadline: 1731278382932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:42,937 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:42,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46638 deadline: 1731278382932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:42,937 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:42,937 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:42,937 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:42,938 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:42,938 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:42,938 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:42,962 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-10T22:38:42,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b. 2024-11-10T22:38:42,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b. as already flushing 2024-11-10T22:38:42,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b. 2024-11-10T22:38:42,963 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 665ebb4a1f83b8f4b9dc2b71f1eaf80b, NAME => 'TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:42,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 665ebb4a1f83b8f4b9dc2b71f1eaf80b, NAME => 'TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:42,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 665ebb4a1f83b8f4b9dc2b71f1eaf80b, NAME => 'TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 665ebb4a1f83b8f4b9dc2b71f1eaf80b, NAME => 'TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:43,007 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/A/e9c98768aac84abb9b1076a9e5518e9b as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/A/e9c98768aac84abb9b1076a9e5518e9b 2024-11-10T22:38:43,012 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/B/414179b8c00649aea83c5f77dfce8c58 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/B/414179b8c00649aea83c5f77dfce8c58 2024-11-10T22:38:43,014 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in 665ebb4a1f83b8f4b9dc2b71f1eaf80b/A of 665ebb4a1f83b8f4b9dc2b71f1eaf80b into e9c98768aac84abb9b1076a9e5518e9b(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T22:38:43,014 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 665ebb4a1f83b8f4b9dc2b71f1eaf80b: 2024-11-10T22:38:43,014 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., storeName=665ebb4a1f83b8f4b9dc2b71f1eaf80b/A, priority=10, startTime=1731278322571; duration=0sec 2024-11-10T22:38:43,014 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T22:38:43,014 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 665ebb4a1f83b8f4b9dc2b71f1eaf80b:A 2024-11-10T22:38:43,014 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-11-10T22:38:43,017 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 72006 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-11-10T22:38:43,018 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HStore(1541): 665ebb4a1f83b8f4b9dc2b71f1eaf80b/C is initiating minor compaction (all files) 2024-11-10T22:38:43,018 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 665ebb4a1f83b8f4b9dc2b71f1eaf80b/C in TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b. 2024-11-10T22:38:43,018 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/C/7ecf0e35f2a94aafb73b2a9b2311a14c, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/C/9493540e1fb2421d9402b0c6a988fe04, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/C/63b834ae163b49619c3041236aeeb61b, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/C/987e7e90493940eca649e12e8376c494, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/C/25a2e22f46064e8dba537d1428553558, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/C/b312b7db25014a4cbbb36a0cc71bd30d] into tmpdir=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp, totalSize=70.3 K 2024-11-10T22:38:43,018 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in 665ebb4a1f83b8f4b9dc2b71f1eaf80b/B of 665ebb4a1f83b8f4b9dc2b71f1eaf80b into 414179b8c00649aea83c5f77dfce8c58(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T22:38:43,018 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 665ebb4a1f83b8f4b9dc2b71f1eaf80b: 2024-11-10T22:38:43,018 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., storeName=665ebb4a1f83b8f4b9dc2b71f1eaf80b/B, priority=10, startTime=1731278322571; duration=0sec 2024-11-10T22:38:43,018 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T22:38:43,018 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 665ebb4a1f83b8f4b9dc2b71f1eaf80b:B 2024-11-10T22:38:43,018 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7ecf0e35f2a94aafb73b2a9b2311a14c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731278317234 2024-11-10T22:38:43,019 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9493540e1fb2421d9402b0c6a988fe04, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731278317264 2024-11-10T22:38:43,019 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 63b834ae163b49619c3041236aeeb61b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731278317937 2024-11-10T22:38:43,019 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 987e7e90493940eca649e12e8376c494, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=73, earliestPutTs=1731278318593 2024-11-10T22:38:43,019 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 25a2e22f46064e8dba537d1428553558, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1731278319768 2024-11-10T22:38:43,020 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting b312b7db25014a4cbbb36a0cc71bd30d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1731278322085 2024-11-10T22:38:43,036 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 665ebb4a1f83b8f4b9dc2b71f1eaf80b#C#compaction#243 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T22:38:43,037 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/C/755408b8418b4b57a8bf59a80d01f098 is 50, key is test_row_0/C:col10/1731278322116/Put/seqid=0 2024-11-10T22:38:43,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742118_1294 (size=12207) 2024-11-10T22:38:43,066 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/C/755408b8418b4b57a8bf59a80d01f098 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/C/755408b8418b4b57a8bf59a80d01f098 2024-11-10T22:38:43,077 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in 665ebb4a1f83b8f4b9dc2b71f1eaf80b/C of 665ebb4a1f83b8f4b9dc2b71f1eaf80b into 755408b8418b4b57a8bf59a80d01f098(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T22:38:43,079 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 665ebb4a1f83b8f4b9dc2b71f1eaf80b: 2024-11-10T22:38:43,079 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., storeName=665ebb4a1f83b8f4b9dc2b71f1eaf80b/C, priority=10, startTime=1731278322571; duration=0sec 2024-11-10T22:38:43,079 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T22:38:43,079 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 665ebb4a1f83b8f4b9dc2b71f1eaf80b:C 2024-11-10T22:38:43,116 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-10T22:38:43,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b. 2024-11-10T22:38:43,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b. as already flushing 2024-11-10T22:38:43,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b. 2024-11-10T22:38:43,116 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 665ebb4a1f83b8f4b9dc2b71f1eaf80b, NAME => 'TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:43,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 665ebb4a1f83b8f4b9dc2b71f1eaf80b, NAME => 'TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:43,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 665ebb4a1f83b8f4b9dc2b71f1eaf80b, NAME => 'TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 665ebb4a1f83b8f4b9dc2b71f1eaf80b, NAME => 'TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:43,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-10T22:38:43,136 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:43,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46654 deadline: 1731278383132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:43,136 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:43,136 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:43,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46594 deadline: 1731278383132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:43,136 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:43,136 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:43,136 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:43,137 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:43,137 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:43,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1731278383132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:43,137 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:43,137 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:43,137 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:43,137 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:43,144 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:43,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46606 deadline: 1731278383143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:43,144 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:43,144 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:43,144 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:43,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46638 deadline: 1731278383143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:43,144 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:43,145 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:43,145 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:43,145 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:43,201 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=128 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/B/b9ef504fcdc74e62a9fc47cc64ec5e0b 2024-11-10T22:38:43,209 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/C/7334e6d4fce0405a92d9edb48d45708d is 50, key is test_row_0/C:col10/1731278322765/Put/seqid=0 2024-11-10T22:38:43,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742119_1295 (size=9657) 2024-11-10T22:38:43,270 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-10T22:38:43,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b. 2024-11-10T22:38:43,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b. as already flushing 2024-11-10T22:38:43,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b. 2024-11-10T22:38:43,270 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 665ebb4a1f83b8f4b9dc2b71f1eaf80b, NAME => 'TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:43,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 665ebb4a1f83b8f4b9dc2b71f1eaf80b, NAME => 'TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:43,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 665ebb4a1f83b8f4b9dc2b71f1eaf80b, NAME => 'TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 665ebb4a1f83b8f4b9dc2b71f1eaf80b, NAME => 'TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:43,423 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-10T22:38:43,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b. 2024-11-10T22:38:43,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b. as already flushing 2024-11-10T22:38:43,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b. 2024-11-10T22:38:43,423 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 665ebb4a1f83b8f4b9dc2b71f1eaf80b, NAME => 'TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:43,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 665ebb4a1f83b8f4b9dc2b71f1eaf80b, NAME => 'TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:43,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 665ebb4a1f83b8f4b9dc2b71f1eaf80b, NAME => 'TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 665ebb4a1f83b8f4b9dc2b71f1eaf80b, NAME => 'TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:43,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-10T22:38:43,444 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:43,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46594 deadline: 1731278383442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:43,445 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:43,445 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:43,444 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:43,445 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:43,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46654 deadline: 1731278383442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:43,445 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:43,445 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:43,445 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:43,445 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:43,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1731278383442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:43,445 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:43,446 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:43,446 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:43,454 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:43,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46606 deadline: 1731278383453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:43,455 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:43,455 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:43,455 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:43,455 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:43,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46638 deadline: 1731278383453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:43,455 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:43,455 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:43,455 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:43,576 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-10T22:38:43,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b. 2024-11-10T22:38:43,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b. as already flushing 2024-11-10T22:38:43,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b. 2024-11-10T22:38:43,577 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 665ebb4a1f83b8f4b9dc2b71f1eaf80b, NAME => 'TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:43,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 665ebb4a1f83b8f4b9dc2b71f1eaf80b, NAME => 'TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:43,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 665ebb4a1f83b8f4b9dc2b71f1eaf80b, NAME => 'TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 665ebb4a1f83b8f4b9dc2b71f1eaf80b, NAME => 'TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:43,615 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=128 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/C/7334e6d4fce0405a92d9edb48d45708d 2024-11-10T22:38:43,620 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/A/63032a8b961a47c9b5fcecee5480b6a0 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/A/63032a8b961a47c9b5fcecee5480b6a0 2024-11-10T22:38:43,624 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/A/63032a8b961a47c9b5fcecee5480b6a0, entries=150, sequenceid=128, filesize=11.7 K 2024-11-10T22:38:43,625 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/B/b9ef504fcdc74e62a9fc47cc64ec5e0b as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/B/b9ef504fcdc74e62a9fc47cc64ec5e0b 2024-11-10T22:38:43,629 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/B/b9ef504fcdc74e62a9fc47cc64ec5e0b, entries=100, sequenceid=128, filesize=9.4 K 2024-11-10T22:38:43,630 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/C/7334e6d4fce0405a92d9edb48d45708d as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/C/7334e6d4fce0405a92d9edb48d45708d 2024-11-10T22:38:43,633 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/C/7334e6d4fce0405a92d9edb48d45708d, entries=100, sequenceid=128, filesize=9.4 K 2024-11-10T22:38:43,634 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 665ebb4a1f83b8f4b9dc2b71f1eaf80b in 868ms, sequenceid=128, compaction requested=false 2024-11-10T22:38:43,634 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 665ebb4a1f83b8f4b9dc2b71f1eaf80b: 2024-11-10T22:38:43,730 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-10T22:38:43,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b. 2024-11-10T22:38:43,730 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing 665ebb4a1f83b8f4b9dc2b71f1eaf80b 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-10T22:38:43,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/A/264503b7d4774771b53806205d18cb90 is 50, key is test_row_0/A:col10/1731278322809/Put/seqid=0 2024-11-10T22:38:43,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742120_1296 (size=12151) 2024-11-10T22:38:43,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-10T22:38:43,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(8855): Flush requested on 665ebb4a1f83b8f4b9dc2b71f1eaf80b 2024-11-10T22:38:43,956 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b. as already flushing 2024-11-10T22:38:43,973 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:43,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46606 deadline: 1731278383967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:43,973 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:43,973 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:43,973 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:43,973 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:43,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46638 deadline: 1731278383967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:43,973 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:43,973 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:43,974 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:43,974 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:43,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46654 deadline: 1731278383968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:43,974 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:43,974 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:43,974 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:43,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1731278383969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:43,974 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:43,974 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:43,974 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:43,974 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:43,976 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:43,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46594 deadline: 1731278383972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:43,977 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:43,977 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:43,977 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:44,086 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:44,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46606 deadline: 1731278384082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:44,086 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:44,086 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:44,086 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:44,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46638 deadline: 1731278384083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:44,086 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:44,086 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:44,086 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:44,087 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:44,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46654 deadline: 1731278384083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:44,087 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:44,087 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:44,087 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:44,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1731278384083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:44,087 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:44,087 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:44,087 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:44,087 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:44,087 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:44,087 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:44,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46594 deadline: 1731278384083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:44,087 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:44,087 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:44,087 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:44,141 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=152 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/A/264503b7d4774771b53806205d18cb90 2024-11-10T22:38:44,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/B/a72051362a6748a4b90ced820fa6ed71 is 50, key is test_row_0/B:col10/1731278322809/Put/seqid=0 2024-11-10T22:38:44,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742121_1297 (size=12151) 2024-11-10T22:38:44,296 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:44,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46638 deadline: 1731278384293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:44,296 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:44,296 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:44,296 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:44,296 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:44,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46606 deadline: 1731278384293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:44,297 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:44,297 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:44,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46654 deadline: 1731278384293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:44,297 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:44,297 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:44,297 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:44,297 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:44,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1731278384293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:44,297 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:44,297 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:44,297 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:44,297 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:44,297 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:44,297 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:44,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46594 deadline: 1731278384293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:44,298 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:44,298 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:44,298 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:44,565 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=152 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/B/a72051362a6748a4b90ced820fa6ed71 2024-11-10T22:38:44,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/C/40b954093197427ab7ddd7358c387eeb is 50, key is test_row_0/C:col10/1731278322809/Put/seqid=0 2024-11-10T22:38:44,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742122_1298 (size=12151) 2024-11-10T22:38:44,606 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:44,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46638 deadline: 1731278384602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:44,606 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:44,606 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:44,607 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:44,607 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:44,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46606 deadline: 1731278384603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:44,607 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:44,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1731278384603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:44,607 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:44,607 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:44,607 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:44,607 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:44,607 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:44,607 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:44,607 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:44,607 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:44,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46654 deadline: 1731278384603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:44,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46594 deadline: 1731278384603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:44,607 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:44,607 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:44,608 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:44,608 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:44,608 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:44,608 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:44,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-10T22:38:44,978 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=152 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/C/40b954093197427ab7ddd7358c387eeb 2024-11-10T22:38:44,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/A/264503b7d4774771b53806205d18cb90 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/A/264503b7d4774771b53806205d18cb90 2024-11-10T22:38:44,988 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/A/264503b7d4774771b53806205d18cb90, entries=150, sequenceid=152, filesize=11.9 K 2024-11-10T22:38:44,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/B/a72051362a6748a4b90ced820fa6ed71 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/B/a72051362a6748a4b90ced820fa6ed71 2024-11-10T22:38:44,993 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/B/a72051362a6748a4b90ced820fa6ed71, entries=150, sequenceid=152, filesize=11.9 K 2024-11-10T22:38:44,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/C/40b954093197427ab7ddd7358c387eeb as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/C/40b954093197427ab7ddd7358c387eeb 2024-11-10T22:38:44,998 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/C/40b954093197427ab7ddd7358c387eeb, entries=150, sequenceid=152, filesize=11.9 K 2024-11-10T22:38:44,999 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for 665ebb4a1f83b8f4b9dc2b71f1eaf80b in 1269ms, sequenceid=152, compaction requested=true 2024-11-10T22:38:44,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for 665ebb4a1f83b8f4b9dc2b71f1eaf80b: 2024-11-10T22:38:45,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b. 2024-11-10T22:38:45,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-11-10T22:38:45,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-11-10T22:38:45,003 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-11-10T22:38:45,003 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1930 sec 2024-11-10T22:38:45,005 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 2.2000 sec 2024-11-10T22:38:45,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(8855): Flush requested on 665ebb4a1f83b8f4b9dc2b71f1eaf80b 2024-11-10T22:38:45,115 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T22:38:45,115 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 665ebb4a1f83b8f4b9dc2b71f1eaf80b 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-10T22:38:45,121 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/A/05ee1128f7d3499793978c982367885b is 50, key is test_row_0/A:col10/1731278325115/Put/seqid=0 2024-11-10T22:38:45,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742123_1299 (size=16931) 2024-11-10T22:38:45,138 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:45,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46638 deadline: 1731278385133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:45,139 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:45,139 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:45,139 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:45,141 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:45,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46594 deadline: 1731278385136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:45,141 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:45,141 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:45,141 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:45,142 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:45,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46606 deadline: 1731278385137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:45,143 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:45,143 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:45,143 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:45,143 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:45,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1731278385137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:45,143 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:45,143 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:45,143 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:45,143 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:45,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46654 deadline: 1731278385138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:45,144 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:45,144 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:45,144 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:45,245 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:45,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46638 deadline: 1731278385242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:45,245 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:45,245 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:45,245 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:45,253 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:45,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46594 deadline: 1731278385252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:45,254 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:45,254 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:45,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46606 deadline: 1731278385252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:45,254 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:45,254 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:45,254 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:45,254 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:45,254 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:45,254 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:45,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1731278385252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:45,255 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:45,255 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:45,255 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:45,255 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:45,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46654 deadline: 1731278385252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:45,255 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:45,255 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:45,255 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:45,455 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:45,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46638 deadline: 1731278385452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:45,456 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:45,456 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:45,456 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:45,466 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:45,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46594 deadline: 1731278385462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:45,466 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:45,466 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:45,466 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:45,466 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:45,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46606 deadline: 1731278385462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:45,466 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:45,467 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:45,467 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:45,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1731278385463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:45,467 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:45,467 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:45,467 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:45,467 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:45,467 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:45,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46654 deadline: 1731278385463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:45,467 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:45,467 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:45,467 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:45,526 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/A/05ee1128f7d3499793978c982367885b 2024-11-10T22:38:45,534 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/B/04698c3651824f2bb23f583068e74c8f is 50, key is test_row_0/B:col10/1731278325115/Put/seqid=0 2024-11-10T22:38:45,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742124_1300 (size=12151) 2024-11-10T22:38:45,767 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:45,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46638 deadline: 1731278385763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:45,767 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:45,767 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:45,767 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:45,773 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:45,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46594 deadline: 1731278385772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:45,774 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:45,773 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:45,774 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:45,774 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:45,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46606 deadline: 1731278385772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:45,774 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:45,774 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:45,774 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:45,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1731278385772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:45,774 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:45,774 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:45,774 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:45,774 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:45,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46654 deadline: 1731278385772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:45,774 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:45,774 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:45,775 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:45,775 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:45,938 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/B/04698c3651824f2bb23f583068e74c8f 2024-11-10T22:38:45,948 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/C/18515a535c9240b394fb00a2dd2474b5 is 50, key is test_row_0/C:col10/1731278325115/Put/seqid=0 2024-11-10T22:38:45,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742125_1301 (size=12151) 2024-11-10T22:38:46,276 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:46,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46638 deadline: 1731278386273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:46,277 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:46,277 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:46,277 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:46,285 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:46,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46594 deadline: 1731278386282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:46,286 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:46,286 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:46,286 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:46,286 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:46,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46606 deadline: 1731278386282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:46,286 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:46,286 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:46,286 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:46,286 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:46,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1731278386282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:46,286 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:46,286 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:46,286 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:46,286 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:46,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46654 deadline: 1731278386282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:46,287 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:46,287 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:46,287 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:46,359 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/C/18515a535c9240b394fb00a2dd2474b5 2024-11-10T22:38:46,363 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/A/05ee1128f7d3499793978c982367885b as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/A/05ee1128f7d3499793978c982367885b 2024-11-10T22:38:46,367 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/A/05ee1128f7d3499793978c982367885b, entries=250, sequenceid=171, filesize=16.5 K 2024-11-10T22:38:46,368 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/B/04698c3651824f2bb23f583068e74c8f as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/B/04698c3651824f2bb23f583068e74c8f 2024-11-10T22:38:46,372 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/B/04698c3651824f2bb23f583068e74c8f, entries=150, sequenceid=171, filesize=11.9 K 2024-11-10T22:38:46,372 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/C/18515a535c9240b394fb00a2dd2474b5 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/C/18515a535c9240b394fb00a2dd2474b5 2024-11-10T22:38:46,376 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/C/18515a535c9240b394fb00a2dd2474b5, entries=150, sequenceid=171, filesize=11.9 K 2024-11-10T22:38:46,377 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for 665ebb4a1f83b8f4b9dc2b71f1eaf80b in 1261ms, sequenceid=171, compaction requested=true 2024-11-10T22:38:46,377 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 665ebb4a1f83b8f4b9dc2b71f1eaf80b: 2024-11-10T22:38:46,377 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 665ebb4a1f83b8f4b9dc2b71f1eaf80b:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T22:38:46,377 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T22:38:46,377 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T22:38:46,377 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 665ebb4a1f83b8f4b9dc2b71f1eaf80b:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T22:38:46,377 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T22:38:46,377 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 665ebb4a1f83b8f4b9dc2b71f1eaf80b:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T22:38:46,377 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T22:38:46,377 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T22:38:46,378 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53286 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T22:38:46,378 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HStore(1541): 665ebb4a1f83b8f4b9dc2b71f1eaf80b/A is initiating minor compaction (all files) 2024-11-10T22:38:46,378 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 665ebb4a1f83b8f4b9dc2b71f1eaf80b/A in TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b. 2024-11-10T22:38:46,378 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/A/e9c98768aac84abb9b1076a9e5518e9b, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/A/63032a8b961a47c9b5fcecee5480b6a0, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/A/264503b7d4774771b53806205d18cb90, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/A/05ee1128f7d3499793978c982367885b] into tmpdir=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp, totalSize=52.0 K 2024-11-10T22:38:46,379 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46166 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T22:38:46,379 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1541): 665ebb4a1f83b8f4b9dc2b71f1eaf80b/B is initiating minor compaction (all files) 2024-11-10T22:38:46,379 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 665ebb4a1f83b8f4b9dc2b71f1eaf80b/B in TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b. 2024-11-10T22:38:46,379 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/B/414179b8c00649aea83c5f77dfce8c58, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/B/b9ef504fcdc74e62a9fc47cc64ec5e0b, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/B/a72051362a6748a4b90ced820fa6ed71, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/B/04698c3651824f2bb23f583068e74c8f] into tmpdir=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp, totalSize=45.1 K 2024-11-10T22:38:46,379 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting e9c98768aac84abb9b1076a9e5518e9b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1731278322085 2024-11-10T22:38:46,379 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting 414179b8c00649aea83c5f77dfce8c58, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1731278322085 2024-11-10T22:38:46,379 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 63032a8b961a47c9b5fcecee5480b6a0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1731278322135 2024-11-10T22:38:46,380 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting b9ef504fcdc74e62a9fc47cc64ec5e0b, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1731278322136 2024-11-10T22:38:46,380 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 264503b7d4774771b53806205d18cb90, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1731278322798 2024-11-10T22:38:46,380 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting a72051362a6748a4b90ced820fa6ed71, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1731278322798 2024-11-10T22:38:46,380 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 05ee1128f7d3499793978c982367885b, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1731278323966 2024-11-10T22:38:46,380 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting 04698c3651824f2bb23f583068e74c8f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1731278323970 2024-11-10T22:38:46,389 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 665ebb4a1f83b8f4b9dc2b71f1eaf80b#A#compaction#251 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T22:38:46,390 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 665ebb4a1f83b8f4b9dc2b71f1eaf80b#B#compaction#252 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T22:38:46,390 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/A/6b83954172ca466f8ac13274d5285c65 is 50, key is test_row_0/A:col10/1731278325115/Put/seqid=0 2024-11-10T22:38:46,390 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/B/750eaf21699f4aaaa164823968eb722d is 50, key is test_row_0/B:col10/1731278325115/Put/seqid=0 2024-11-10T22:38:46,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742126_1302 (size=12493) 2024-11-10T22:38:46,413 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742127_1303 (size=12493) 2024-11-10T22:38:46,420 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/B/750eaf21699f4aaaa164823968eb722d as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/B/750eaf21699f4aaaa164823968eb722d 2024-11-10T22:38:46,426 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 665ebb4a1f83b8f4b9dc2b71f1eaf80b/B of 665ebb4a1f83b8f4b9dc2b71f1eaf80b into 750eaf21699f4aaaa164823968eb722d(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T22:38:46,426 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 665ebb4a1f83b8f4b9dc2b71f1eaf80b: 2024-11-10T22:38:46,426 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., storeName=665ebb4a1f83b8f4b9dc2b71f1eaf80b/B, priority=12, startTime=1731278326377; duration=0sec 2024-11-10T22:38:46,426 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T22:38:46,426 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 665ebb4a1f83b8f4b9dc2b71f1eaf80b:B 2024-11-10T22:38:46,426 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T22:38:46,428 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46166 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T22:38:46,428 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1541): 665ebb4a1f83b8f4b9dc2b71f1eaf80b/C is initiating minor compaction (all files) 2024-11-10T22:38:46,428 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 665ebb4a1f83b8f4b9dc2b71f1eaf80b/C in TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b. 2024-11-10T22:38:46,428 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/C/755408b8418b4b57a8bf59a80d01f098, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/C/7334e6d4fce0405a92d9edb48d45708d, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/C/40b954093197427ab7ddd7358c387eeb, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/C/18515a535c9240b394fb00a2dd2474b5] into tmpdir=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp, totalSize=45.1 K 2024-11-10T22:38:46,428 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting 755408b8418b4b57a8bf59a80d01f098, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1731278322085 2024-11-10T22:38:46,429 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting 7334e6d4fce0405a92d9edb48d45708d, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1731278322136 2024-11-10T22:38:46,429 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting 40b954093197427ab7ddd7358c387eeb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1731278322798 2024-11-10T22:38:46,432 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting 18515a535c9240b394fb00a2dd2474b5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1731278323970 2024-11-10T22:38:46,442 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 665ebb4a1f83b8f4b9dc2b71f1eaf80b#C#compaction#253 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T22:38:46,442 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/C/b2892919e0844e1299854f4f16fe4742 is 50, key is test_row_0/C:col10/1731278325115/Put/seqid=0 2024-11-10T22:38:46,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742128_1304 (size=12493) 2024-11-10T22:38:46,465 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/C/b2892919e0844e1299854f4f16fe4742 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/C/b2892919e0844e1299854f4f16fe4742 2024-11-10T22:38:46,471 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 665ebb4a1f83b8f4b9dc2b71f1eaf80b/C of 665ebb4a1f83b8f4b9dc2b71f1eaf80b into b2892919e0844e1299854f4f16fe4742(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T22:38:46,471 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 665ebb4a1f83b8f4b9dc2b71f1eaf80b: 2024-11-10T22:38:46,471 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., storeName=665ebb4a1f83b8f4b9dc2b71f1eaf80b/C, priority=12, startTime=1731278326377; duration=0sec 2024-11-10T22:38:46,471 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T22:38:46,471 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 665ebb4a1f83b8f4b9dc2b71f1eaf80b:C 2024-11-10T22:38:46,818 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/A/6b83954172ca466f8ac13274d5285c65 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/A/6b83954172ca466f8ac13274d5285c65 2024-11-10T22:38:46,823 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 665ebb4a1f83b8f4b9dc2b71f1eaf80b/A of 665ebb4a1f83b8f4b9dc2b71f1eaf80b into 6b83954172ca466f8ac13274d5285c65(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T22:38:46,823 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 665ebb4a1f83b8f4b9dc2b71f1eaf80b: 2024-11-10T22:38:46,823 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., storeName=665ebb4a1f83b8f4b9dc2b71f1eaf80b/A, priority=12, startTime=1731278326377; duration=0sec 2024-11-10T22:38:46,823 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T22:38:46,823 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 665ebb4a1f83b8f4b9dc2b71f1eaf80b:A 2024-11-10T22:38:46,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-10T22:38:46,973 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T22:38:46,973 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T22:38:46,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees 2024-11-10T22:38:46,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-10T22:38:46,976 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T22:38:46,976 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T22:38:46,977 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T22:38:47,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-10T22:38:47,129 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-10T22:38:47,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b. 2024-11-10T22:38:47,129 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2902): Flushing 665ebb4a1f83b8f4b9dc2b71f1eaf80b 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-10T22:38:47,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/A/1a8acacf8d4f4ebbbcb91da594c0d77e is 50, key is test_row_0/A:col10/1731278325127/Put/seqid=0 2024-11-10T22:38:47,139 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742129_1305 (size=12151) 2024-11-10T22:38:47,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(8855): Flush requested on 665ebb4a1f83b8f4b9dc2b71f1eaf80b 2024-11-10T22:38:47,285 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b. as already flushing 2024-11-10T22:38:47,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-10T22:38:47,310 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:47,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46606 deadline: 1731278387305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:47,310 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:47,310 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:47,310 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:47,311 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:47,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1731278387306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:47,311 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:47,311 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:47,311 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:47,311 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:47,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46638 deadline: 1731278387307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:47,312 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:47,312 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:47,312 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:47,312 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:47,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46594 deadline: 1731278387308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:47,312 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:47,312 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:47,312 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:47,314 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:47,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46654 deadline: 1731278387310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:47,314 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:47,315 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:47,315 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:47,414 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:47,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46606 deadline: 1731278387413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:47,415 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:47,415 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:47,415 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:47,423 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:47,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1731278387422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:47,424 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:47,424 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:47,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46638 deadline: 1731278387422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:47,424 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:47,424 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:47,424 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:47,424 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:47,424 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:47,425 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:47,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46594 deadline: 1731278387422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:47,426 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:47,426 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:47,426 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:47,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46654 deadline: 1731278387422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:47,426 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:47,426 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:47,426 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:47,426 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:47,539 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=191 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/A/1a8acacf8d4f4ebbbcb91da594c0d77e 2024-11-10T22:38:47,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/B/8c88bbb4edd64b3d8a998fd5c2a24663 is 50, key is test_row_0/B:col10/1731278325127/Put/seqid=0 2024-11-10T22:38:47,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742130_1306 (size=12151) 2024-11-10T22:38:47,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-10T22:38:47,626 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:47,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46606 deadline: 1731278387622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:47,626 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:47,626 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:47,626 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:47,635 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:47,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1731278387632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:47,635 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:47,635 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:47,635 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:47,635 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:47,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46638 deadline: 1731278387632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:47,635 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:47,635 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:47,636 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:47,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46594 deadline: 1731278387632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:47,636 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:47,636 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:47,636 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:47,636 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:47,636 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:47,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46654 deadline: 1731278387633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:47,636 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:47,636 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:47,636 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:47,934 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:47,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46606 deadline: 1731278387932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:47,934 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:47,934 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:47,935 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:47,945 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:47,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1731278387942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:47,945 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:47,946 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:47,946 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:47,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46638 deadline: 1731278387942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:47,946 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:47,946 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:47,946 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:47,946 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:47,946 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:47,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46594 deadline: 1731278387942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:47,946 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:47,946 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:47,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46654 deadline: 1731278387942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:47,946 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:47,946 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:47,946 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:47,947 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:47,947 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:47,957 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=191 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/B/8c88bbb4edd64b3d8a998fd5c2a24663 2024-11-10T22:38:47,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/C/fd7b0934729341588f79e18efcd0a831 is 50, key is test_row_0/C:col10/1731278325127/Put/seqid=0 2024-11-10T22:38:47,977 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742131_1307 (size=12151) 2024-11-10T22:38:48,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-10T22:38:48,378 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=191 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/C/fd7b0934729341588f79e18efcd0a831 2024-11-10T22:38:48,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/A/1a8acacf8d4f4ebbbcb91da594c0d77e as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/A/1a8acacf8d4f4ebbbcb91da594c0d77e 2024-11-10T22:38:48,387 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/A/1a8acacf8d4f4ebbbcb91da594c0d77e, entries=150, sequenceid=191, filesize=11.9 K 2024-11-10T22:38:48,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/B/8c88bbb4edd64b3d8a998fd5c2a24663 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/B/8c88bbb4edd64b3d8a998fd5c2a24663 2024-11-10T22:38:48,392 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/B/8c88bbb4edd64b3d8a998fd5c2a24663, entries=150, sequenceid=191, filesize=11.9 K 2024-11-10T22:38:48,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/C/fd7b0934729341588f79e18efcd0a831 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/C/fd7b0934729341588f79e18efcd0a831 2024-11-10T22:38:48,396 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/C/fd7b0934729341588f79e18efcd0a831, entries=150, sequenceid=191, filesize=11.9 K 2024-11-10T22:38:48,397 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for 665ebb4a1f83b8f4b9dc2b71f1eaf80b in 1268ms, sequenceid=191, compaction requested=false 2024-11-10T22:38:48,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2603): Flush status journal for 665ebb4a1f83b8f4b9dc2b71f1eaf80b: 2024-11-10T22:38:48,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b. 2024-11-10T22:38:48,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=77 2024-11-10T22:38:48,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.HMaster(4169): Remote procedure done, pid=77 2024-11-10T22:38:48,400 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-11-10T22:38:48,400 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4220 sec 2024-11-10T22:38:48,402 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees in 1.4280 sec 2024-11-10T22:38:48,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(8855): Flush requested on 665ebb4a1f83b8f4b9dc2b71f1eaf80b 2024-11-10T22:38:48,446 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T22:38:48,446 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 665ebb4a1f83b8f4b9dc2b71f1eaf80b 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-10T22:38:48,451 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/A/907da03b1de040bdb1bd8406609e1f42 is 50, key is test_row_0/A:col10/1731278328444/Put/seqid=0 2024-11-10T22:38:48,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742132_1308 (size=14541) 2024-11-10T22:38:48,469 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:48,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46638 deadline: 1731278388463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:48,469 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:48,469 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:48,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46594 deadline: 1731278388464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:48,469 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:48,469 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:48,469 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:48,469 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:48,470 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:48,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46654 deadline: 1731278388465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:48,470 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:48,470 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:48,470 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:48,470 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:48,470 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:48,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46606 deadline: 1731278388466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:48,471 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:48,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1731278388466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:48,471 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:48,471 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:48,471 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:48,471 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:48,471 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:48,471 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:48,574 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:48,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46638 deadline: 1731278388573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:48,574 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:48,575 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:48,575 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:48,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46594 deadline: 1731278388573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:48,575 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:48,575 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:48,575 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:48,575 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:48,575 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:48,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46654 deadline: 1731278388573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:48,575 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:48,575 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:48,575 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:48,575 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:48,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46606 deadline: 1731278388573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:48,575 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:48,575 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:48,575 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:48,586 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:48,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1731278388582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:48,586 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:48,586 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:48,586 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:48,786 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:48,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46638 deadline: 1731278388782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:48,786 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:48,786 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:48,786 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:48,787 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:48,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46594 deadline: 1731278388782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:48,787 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:48,787 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:48,787 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:48,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46654 deadline: 1731278388782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:48,787 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:48,787 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:48,787 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:48,787 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:48,787 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:48,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46606 deadline: 1731278388782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:48,787 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:48,788 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:48,788 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:48,796 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:48,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1731278388793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:48,797 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:48,797 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:48,797 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:48,856 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/A/907da03b1de040bdb1bd8406609e1f42 2024-11-10T22:38:48,864 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/B/ddea2f2255dd4bb19eb2d1ee6dba5d09 is 50, key is test_row_0/B:col10/1731278328444/Put/seqid=0 2024-11-10T22:38:48,867 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742133_1309 (size=12151) 2024-11-10T22:38:49,093 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:49,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46638 deadline: 1731278389092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:49,094 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:49,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46594 deadline: 1731278389092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:49,094 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:49,094 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:49,094 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:49,094 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:49,094 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:49,094 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:49,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46654 deadline: 1731278389092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:49,094 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:49,094 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:49,094 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:49,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46606 deadline: 1731278389092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:49,094 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:49,094 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:49,094 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:49,094 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:49,094 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:49,105 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:49,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1731278389102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:49,106 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:49,106 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:49,106 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:49,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-10T22:38:49,122 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T22:38:49,123 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T22:38:49,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees 2024-11-10T22:38:49,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-10T22:38:49,125 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T22:38:49,126 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T22:38:49,126 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T22:38:49,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-10T22:38:49,268 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/B/ddea2f2255dd4bb19eb2d1ee6dba5d09 2024-11-10T22:38:49,276 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/C/a12edef64f4a427f8638470ec07c6b45 is 50, key is test_row_0/C:col10/1731278328444/Put/seqid=0 2024-11-10T22:38:49,278 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-10T22:38:49,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b. 2024-11-10T22:38:49,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b. as already flushing 2024-11-10T22:38:49,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b. 2024-11-10T22:38:49,279 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 665ebb4a1f83b8f4b9dc2b71f1eaf80b, NAME => 'TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:49,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 665ebb4a1f83b8f4b9dc2b71f1eaf80b, NAME => 'TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:49,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 665ebb4a1f83b8f4b9dc2b71f1eaf80b, NAME => 'TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 665ebb4a1f83b8f4b9dc2b71f1eaf80b, NAME => 'TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:49,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742134_1310 (size=12151) 2024-11-10T22:38:49,432 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-10T22:38:49,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b. 2024-11-10T22:38:49,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b. as already flushing 2024-11-10T22:38:49,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b. 2024-11-10T22:38:49,433 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 665ebb4a1f83b8f4b9dc2b71f1eaf80b, NAME => 'TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:49,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 665ebb4a1f83b8f4b9dc2b71f1eaf80b, NAME => 'TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:49,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 665ebb4a1f83b8f4b9dc2b71f1eaf80b, NAME => 'TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 665ebb4a1f83b8f4b9dc2b71f1eaf80b, NAME => 'TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:49,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-10T22:38:49,586 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-10T22:38:49,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b. 2024-11-10T22:38:49,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b. as already flushing 2024-11-10T22:38:49,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b. 2024-11-10T22:38:49,586 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 665ebb4a1f83b8f4b9dc2b71f1eaf80b, NAME => 'TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:49,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 665ebb4a1f83b8f4b9dc2b71f1eaf80b, NAME => 'TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:49,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 665ebb4a1f83b8f4b9dc2b71f1eaf80b, NAME => 'TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 665ebb4a1f83b8f4b9dc2b71f1eaf80b, NAME => 'TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:49,603 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:49,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46638 deadline: 1731278389603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:49,604 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:49,604 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:49,604 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:49,607 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:49,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46606 deadline: 1731278389603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:49,607 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:49,607 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:49,607 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:49,607 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:49,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46594 deadline: 1731278389603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:49,607 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:49,607 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:49,608 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:49,608 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:49,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46654 deadline: 1731278389603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:49,608 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:49,608 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:49,608 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:49,613 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:49,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1731278389612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:49,613 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:49,613 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:49,613 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:49,685 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/C/a12edef64f4a427f8638470ec07c6b45 2024-11-10T22:38:49,691 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/A/907da03b1de040bdb1bd8406609e1f42 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/A/907da03b1de040bdb1bd8406609e1f42 2024-11-10T22:38:49,695 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/A/907da03b1de040bdb1bd8406609e1f42, entries=200, sequenceid=211, filesize=14.2 K 2024-11-10T22:38:49,697 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/B/ddea2f2255dd4bb19eb2d1ee6dba5d09 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/B/ddea2f2255dd4bb19eb2d1ee6dba5d09 2024-11-10T22:38:49,705 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/B/ddea2f2255dd4bb19eb2d1ee6dba5d09, entries=150, sequenceid=211, filesize=11.9 K 2024-11-10T22:38:49,707 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/C/a12edef64f4a427f8638470ec07c6b45 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/C/a12edef64f4a427f8638470ec07c6b45 2024-11-10T22:38:49,711 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/C/a12edef64f4a427f8638470ec07c6b45, entries=150, sequenceid=211, filesize=11.9 K 2024-11-10T22:38:49,715 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for 665ebb4a1f83b8f4b9dc2b71f1eaf80b in 1269ms, sequenceid=211, compaction requested=true 2024-11-10T22:38:49,716 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 665ebb4a1f83b8f4b9dc2b71f1eaf80b: 2024-11-10T22:38:49,716 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T22:38:49,716 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 665ebb4a1f83b8f4b9dc2b71f1eaf80b:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T22:38:49,716 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T22:38:49,716 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 665ebb4a1f83b8f4b9dc2b71f1eaf80b:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T22:38:49,716 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T22:38:49,716 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 665ebb4a1f83b8f4b9dc2b71f1eaf80b:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T22:38:49,716 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T22:38:49,716 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T22:38:49,718 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T22:38:49,718 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39185 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T22:38:49,718 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HStore(1541): 665ebb4a1f83b8f4b9dc2b71f1eaf80b/A is initiating minor compaction (all files) 2024-11-10T22:38:49,718 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1541): 665ebb4a1f83b8f4b9dc2b71f1eaf80b/B is initiating minor compaction (all files) 2024-11-10T22:38:49,718 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 665ebb4a1f83b8f4b9dc2b71f1eaf80b/A in TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b. 2024-11-10T22:38:49,718 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 665ebb4a1f83b8f4b9dc2b71f1eaf80b/B in TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b. 2024-11-10T22:38:49,718 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/A/6b83954172ca466f8ac13274d5285c65, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/A/1a8acacf8d4f4ebbbcb91da594c0d77e, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/A/907da03b1de040bdb1bd8406609e1f42] into tmpdir=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp, totalSize=38.3 K 2024-11-10T22:38:49,718 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/B/750eaf21699f4aaaa164823968eb722d, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/B/8c88bbb4edd64b3d8a998fd5c2a24663, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/B/ddea2f2255dd4bb19eb2d1ee6dba5d09] into tmpdir=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp, totalSize=35.9 K 2024-11-10T22:38:49,718 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting 750eaf21699f4aaaa164823968eb722d, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1731278323970 2024-11-10T22:38:49,718 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6b83954172ca466f8ac13274d5285c65, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1731278323970 2024-11-10T22:38:49,719 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting 8c88bbb4edd64b3d8a998fd5c2a24663, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1731278325127 2024-11-10T22:38:49,719 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1a8acacf8d4f4ebbbcb91da594c0d77e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1731278325127 2024-11-10T22:38:49,719 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting ddea2f2255dd4bb19eb2d1ee6dba5d09, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1731278327305 2024-11-10T22:38:49,719 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 907da03b1de040bdb1bd8406609e1f42, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1731278327305 2024-11-10T22:38:49,733 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 665ebb4a1f83b8f4b9dc2b71f1eaf80b#B#compaction#260 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T22:38:49,733 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/B/0c741fe94efb424aa5ca9d5dfd76b3df is 50, key is test_row_0/B:col10/1731278328444/Put/seqid=0 2024-11-10T22:38:49,736 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 665ebb4a1f83b8f4b9dc2b71f1eaf80b#A#compaction#261 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T22:38:49,737 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/A/aa73af33ebfc4e8591c8b6bf9568cac8 is 50, key is test_row_0/A:col10/1731278328444/Put/seqid=0 2024-11-10T22:38:49,740 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-10T22:38:49,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b. 2024-11-10T22:38:49,740 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2902): Flushing 665ebb4a1f83b8f4b9dc2b71f1eaf80b 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-10T22:38:49,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/A/54dd7fea22e94151a760b8b635784d68 is 50, key is test_row_0/A:col10/1731278328465/Put/seqid=0 2024-11-10T22:38:49,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-10T22:38:49,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742135_1311 (size=12595) 2024-11-10T22:38:49,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742136_1312 (size=12595) 2024-11-10T22:38:49,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742137_1313 (size=12151) 2024-11-10T22:38:49,771 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/B/0c741fe94efb424aa5ca9d5dfd76b3df as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/B/0c741fe94efb424aa5ca9d5dfd76b3df 2024-11-10T22:38:49,772 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/A/aa73af33ebfc4e8591c8b6bf9568cac8 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/A/aa73af33ebfc4e8591c8b6bf9568cac8 2024-11-10T22:38:49,788 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 665ebb4a1f83b8f4b9dc2b71f1eaf80b/A of 665ebb4a1f83b8f4b9dc2b71f1eaf80b into aa73af33ebfc4e8591c8b6bf9568cac8(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T22:38:49,788 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 665ebb4a1f83b8f4b9dc2b71f1eaf80b/B of 665ebb4a1f83b8f4b9dc2b71f1eaf80b into 0c741fe94efb424aa5ca9d5dfd76b3df(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T22:38:49,789 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 665ebb4a1f83b8f4b9dc2b71f1eaf80b: 2024-11-10T22:38:49,789 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 665ebb4a1f83b8f4b9dc2b71f1eaf80b: 2024-11-10T22:38:49,789 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., storeName=665ebb4a1f83b8f4b9dc2b71f1eaf80b/A, priority=13, startTime=1731278329716; duration=0sec 2024-11-10T22:38:49,789 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., storeName=665ebb4a1f83b8f4b9dc2b71f1eaf80b/B, priority=13, startTime=1731278329716; duration=0sec 2024-11-10T22:38:49,789 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T22:38:49,789 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T22:38:49,789 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 665ebb4a1f83b8f4b9dc2b71f1eaf80b:A 2024-11-10T22:38:49,789 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 665ebb4a1f83b8f4b9dc2b71f1eaf80b:B 2024-11-10T22:38:49,789 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T22:38:49,790 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T22:38:49,790 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HStore(1541): 665ebb4a1f83b8f4b9dc2b71f1eaf80b/C is initiating minor compaction (all files) 2024-11-10T22:38:49,790 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 665ebb4a1f83b8f4b9dc2b71f1eaf80b/C in TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b. 2024-11-10T22:38:49,790 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/C/b2892919e0844e1299854f4f16fe4742, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/C/fd7b0934729341588f79e18efcd0a831, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/C/a12edef64f4a427f8638470ec07c6b45] into tmpdir=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp, totalSize=35.9 K 2024-11-10T22:38:49,790 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting b2892919e0844e1299854f4f16fe4742, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1731278323970 2024-11-10T22:38:49,791 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting fd7b0934729341588f79e18efcd0a831, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1731278325127 2024-11-10T22:38:49,791 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting a12edef64f4a427f8638470ec07c6b45, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1731278327305 2024-11-10T22:38:49,803 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 665ebb4a1f83b8f4b9dc2b71f1eaf80b#C#compaction#263 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T22:38:49,803 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/C/3c7ded45110944859af8bafe93df5cae is 50, key is test_row_0/C:col10/1731278328444/Put/seqid=0 2024-11-10T22:38:49,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742138_1314 (size=12595) 2024-11-10T22:38:50,172 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=227 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/A/54dd7fea22e94151a760b8b635784d68 2024-11-10T22:38:50,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/B/52fc2a922a0b48858a69e505949cd5b7 is 50, key is test_row_0/B:col10/1731278328465/Put/seqid=0 2024-11-10T22:38:50,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742139_1315 (size=12151) 2024-11-10T22:38:50,185 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=227 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/B/52fc2a922a0b48858a69e505949cd5b7 2024-11-10T22:38:50,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/C/60115c271ed145a7a381e54f0b9da706 is 50, key is test_row_0/C:col10/1731278328465/Put/seqid=0 2024-11-10T22:38:50,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742140_1316 (size=12151) 2024-11-10T22:38:50,201 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=227 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/C/60115c271ed145a7a381e54f0b9da706 2024-11-10T22:38:50,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/A/54dd7fea22e94151a760b8b635784d68 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/A/54dd7fea22e94151a760b8b635784d68 2024-11-10T22:38:50,210 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/A/54dd7fea22e94151a760b8b635784d68, entries=150, sequenceid=227, filesize=11.9 K 2024-11-10T22:38:50,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/B/52fc2a922a0b48858a69e505949cd5b7 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/B/52fc2a922a0b48858a69e505949cd5b7 2024-11-10T22:38:50,216 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/C/3c7ded45110944859af8bafe93df5cae as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/C/3c7ded45110944859af8bafe93df5cae 2024-11-10T22:38:50,218 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/B/52fc2a922a0b48858a69e505949cd5b7, entries=150, sequenceid=227, filesize=11.9 K 2024-11-10T22:38:50,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/C/60115c271ed145a7a381e54f0b9da706 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/C/60115c271ed145a7a381e54f0b9da706 2024-11-10T22:38:50,221 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 665ebb4a1f83b8f4b9dc2b71f1eaf80b/C of 665ebb4a1f83b8f4b9dc2b71f1eaf80b into 3c7ded45110944859af8bafe93df5cae(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T22:38:50,221 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 665ebb4a1f83b8f4b9dc2b71f1eaf80b: 2024-11-10T22:38:50,221 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., storeName=665ebb4a1f83b8f4b9dc2b71f1eaf80b/C, priority=13, startTime=1731278329716; duration=0sec 2024-11-10T22:38:50,221 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T22:38:50,221 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 665ebb4a1f83b8f4b9dc2b71f1eaf80b:C 2024-11-10T22:38:50,223 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/C/60115c271ed145a7a381e54f0b9da706, entries=150, sequenceid=227, filesize=11.9 K 2024-11-10T22:38:50,224 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=0 B/0 for 665ebb4a1f83b8f4b9dc2b71f1eaf80b in 484ms, sequenceid=227, compaction requested=false 2024-11-10T22:38:50,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2603): Flush status journal for 665ebb4a1f83b8f4b9dc2b71f1eaf80b: 2024-11-10T22:38:50,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b. 2024-11-10T22:38:50,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=79 2024-11-10T22:38:50,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.HMaster(4169): Remote procedure done, pid=79 2024-11-10T22:38:50,229 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-11-10T22:38:50,229 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1000 sec 2024-11-10T22:38:50,232 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees in 1.1060 sec 2024-11-10T22:38:50,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-10T22:38:50,262 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T22:38:50,263 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T22:38:50,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees 2024-11-10T22:38:50,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-10T22:38:50,265 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T22:38:50,266 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T22:38:50,266 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T22:38:50,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-10T22:38:50,419 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-10T22:38:50,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b. 2024-11-10T22:38:50,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2603): Flush status journal for 665ebb4a1f83b8f4b9dc2b71f1eaf80b: 2024-11-10T22:38:50,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b. 2024-11-10T22:38:50,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=81 2024-11-10T22:38:50,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.HMaster(4169): Remote procedure done, pid=81 2024-11-10T22:38:50,423 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-11-10T22:38:50,423 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 155 msec 2024-11-10T22:38:50,426 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees in 161 msec 2024-11-10T22:38:50,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-10T22:38:50,583 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T22:38:50,583 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T22:38:50,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] procedure2.ProcedureExecutor(1139): Stored pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees 2024-11-10T22:38:50,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-10T22:38:50,586 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T22:38:50,587 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T22:38:50,587 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T22:38:50,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(8855): Flush requested on 665ebb4a1f83b8f4b9dc2b71f1eaf80b 2024-11-10T22:38:50,622 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T22:38:50,622 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 665ebb4a1f83b8f4b9dc2b71f1eaf80b 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-10T22:38:50,633 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/A/3c663e4a96e24a4cbb08724a4e3eaae8 is 50, key is test_row_0/A:col10/1731278330616/Put/seqid=0 2024-11-10T22:38:50,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742141_1317 (size=14541) 2024-11-10T22:38:50,657 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:50,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46654 deadline: 1731278390652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:50,658 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:50,658 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:50,658 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:50,658 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:50,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46606 deadline: 1731278390653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:50,658 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:50,658 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:50,658 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:50,659 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:50,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1731278390654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:50,660 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:50,660 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:50,660 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:50,660 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:50,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46638 deadline: 1731278390655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:50,660 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:50,660 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:50,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46594 deadline: 1731278390657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:50,660 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:50,660 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:50,660 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:50,661 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:50,661 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:50,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-10T22:38:50,740 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-10T22:38:50,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b. 2024-11-10T22:38:50,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b. as already flushing 2024-11-10T22:38:50,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b. 2024-11-10T22:38:50,740 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 665ebb4a1f83b8f4b9dc2b71f1eaf80b, NAME => 'TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:50,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 665ebb4a1f83b8f4b9dc2b71f1eaf80b, NAME => 'TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:50,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 665ebb4a1f83b8f4b9dc2b71f1eaf80b, NAME => 'TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 665ebb4a1f83b8f4b9dc2b71f1eaf80b, NAME => 'TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:50,766 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:50,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46654 deadline: 1731278390763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:50,766 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:50,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46606 deadline: 1731278390763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:50,767 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:50,767 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:50,767 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:50,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1731278390763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:50,767 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:50,766 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:50,767 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:50,767 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:50,767 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:50,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46638 deadline: 1731278390763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:50,767 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:50,767 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:50,767 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:50,768 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:50,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46594 deadline: 1731278390763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:50,768 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:50,768 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:50,768 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:50,768 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:50,768 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:50,768 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:50,893 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-10T22:38:50,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b. 2024-11-10T22:38:50,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b. as already flushing 2024-11-10T22:38:50,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b. 2024-11-10T22:38:50,894 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 665ebb4a1f83b8f4b9dc2b71f1eaf80b, NAME => 'TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:50,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 665ebb4a1f83b8f4b9dc2b71f1eaf80b, NAME => 'TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:50,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 665ebb4a1f83b8f4b9dc2b71f1eaf80b, NAME => 'TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 665ebb4a1f83b8f4b9dc2b71f1eaf80b, NAME => 'TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:50,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-10T22:38:50,975 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:50,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46606 deadline: 1731278390973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:50,976 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:50,976 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:50,976 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:50,976 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:50,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46654 deadline: 1731278390973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:50,976 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:50,976 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:50,976 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:50,976 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:50,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46638 deadline: 1731278390973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:50,977 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:50,977 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:50,977 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:50,977 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:50,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46594 deadline: 1731278390973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:50,977 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:50,977 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:50,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1731278390973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:50,977 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:50,977 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:50,978 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:50,978 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:50,978 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:51,044 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=242 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/A/3c663e4a96e24a4cbb08724a4e3eaae8 2024-11-10T22:38:51,047 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-10T22:38:51,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b. 2024-11-10T22:38:51,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b. as already flushing 2024-11-10T22:38:51,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b. 2024-11-10T22:38:51,047 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 665ebb4a1f83b8f4b9dc2b71f1eaf80b, NAME => 'TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:51,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 665ebb4a1f83b8f4b9dc2b71f1eaf80b, NAME => 'TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:51,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 665ebb4a1f83b8f4b9dc2b71f1eaf80b, NAME => 'TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 665ebb4a1f83b8f4b9dc2b71f1eaf80b, NAME => 'TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:51,052 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/B/f9e6f5f6884d45aba56fcb86bacb8859 is 50, key is test_row_0/B:col10/1731278330616/Put/seqid=0 2024-11-10T22:38:51,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742142_1318 (size=12151) 2024-11-10T22:38:51,200 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-10T22:38:51,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b. 2024-11-10T22:38:51,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b. as already flushing 2024-11-10T22:38:51,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b. 2024-11-10T22:38:51,200 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 665ebb4a1f83b8f4b9dc2b71f1eaf80b, NAME => 'TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:51,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 665ebb4a1f83b8f4b9dc2b71f1eaf80b, NAME => 'TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:51,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 665ebb4a1f83b8f4b9dc2b71f1eaf80b, NAME => 'TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 665ebb4a1f83b8f4b9dc2b71f1eaf80b, NAME => 'TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:51,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-10T22:38:51,286 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:51,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46606 deadline: 1731278391283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:51,287 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:51,287 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:51,287 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:51,287 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:51,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46654 deadline: 1731278391283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:51,287 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:51,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46638 deadline: 1731278391283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:51,287 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:51,287 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:51,287 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:51,287 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:51,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46594 deadline: 1731278391283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:51,287 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:51,287 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:51,287 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:51,288 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:51,288 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:51,288 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:51,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1731278391283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:51,288 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:51,288 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:51,288 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:51,288 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:51,353 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-10T22:38:51,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b. 2024-11-10T22:38:51,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b. as already flushing 2024-11-10T22:38:51,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b. 2024-11-10T22:38:51,353 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 665ebb4a1f83b8f4b9dc2b71f1eaf80b, NAME => 'TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:51,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 665ebb4a1f83b8f4b9dc2b71f1eaf80b, NAME => 'TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:51,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 665ebb4a1f83b8f4b9dc2b71f1eaf80b, NAME => 'TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 665ebb4a1f83b8f4b9dc2b71f1eaf80b, NAME => 'TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:51,456 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=242 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/B/f9e6f5f6884d45aba56fcb86bacb8859 2024-11-10T22:38:51,463 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/C/a8e296c22c7e43ed8a55a72ca0d7ee40 is 50, key is test_row_0/C:col10/1731278330616/Put/seqid=0 2024-11-10T22:38:51,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742143_1319 (size=12151) 2024-11-10T22:38:51,506 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-10T22:38:51,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b. 2024-11-10T22:38:51,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b. as already flushing 2024-11-10T22:38:51,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b. 2024-11-10T22:38:51,506 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 665ebb4a1f83b8f4b9dc2b71f1eaf80b, NAME => 'TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:51,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 665ebb4a1f83b8f4b9dc2b71f1eaf80b, NAME => 'TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:51,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 665ebb4a1f83b8f4b9dc2b71f1eaf80b, NAME => 'TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 665ebb4a1f83b8f4b9dc2b71f1eaf80b, NAME => 'TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:51,659 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-10T22:38:51,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b. 2024-11-10T22:38:51,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b. as already flushing 2024-11-10T22:38:51,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b. 2024-11-10T22:38:51,660 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 665ebb4a1f83b8f4b9dc2b71f1eaf80b, NAME => 'TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:51,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 665ebb4a1f83b8f4b9dc2b71f1eaf80b, NAME => 'TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:51,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 665ebb4a1f83b8f4b9dc2b71f1eaf80b, NAME => 'TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 665ebb4a1f83b8f4b9dc2b71f1eaf80b, NAME => 'TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:51,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-10T22:38:51,794 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:51,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46606 deadline: 1731278391792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:51,794 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:51,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46654 deadline: 1731278391792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:51,794 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:51,794 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:51,795 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:51,795 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:51,795 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:51,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46638 deadline: 1731278391792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:51,795 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:51,795 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:51,795 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:51,795 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:51,795 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:51,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46594 deadline: 1731278391792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:51,795 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:51,795 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:51,795 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:51,795 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:51,795 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:51,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1731278391792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:51,795 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:51,795 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:51,795 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:51,812 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-10T22:38:51,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b. 2024-11-10T22:38:51,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b. as already flushing 2024-11-10T22:38:51,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b. 2024-11-10T22:38:51,813 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 665ebb4a1f83b8f4b9dc2b71f1eaf80b, NAME => 'TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:51,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 665ebb4a1f83b8f4b9dc2b71f1eaf80b, NAME => 'TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:51,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 665ebb4a1f83b8f4b9dc2b71f1eaf80b, NAME => 'TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 665ebb4a1f83b8f4b9dc2b71f1eaf80b, NAME => 'TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:51,868 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=242 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/C/a8e296c22c7e43ed8a55a72ca0d7ee40 2024-11-10T22:38:51,872 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/A/3c663e4a96e24a4cbb08724a4e3eaae8 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/A/3c663e4a96e24a4cbb08724a4e3eaae8 2024-11-10T22:38:51,876 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/A/3c663e4a96e24a4cbb08724a4e3eaae8, entries=200, sequenceid=242, filesize=14.2 K 2024-11-10T22:38:51,877 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/B/f9e6f5f6884d45aba56fcb86bacb8859 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/B/f9e6f5f6884d45aba56fcb86bacb8859 2024-11-10T22:38:51,882 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/B/f9e6f5f6884d45aba56fcb86bacb8859, entries=150, sequenceid=242, filesize=11.9 K 2024-11-10T22:38:51,883 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/C/a8e296c22c7e43ed8a55a72ca0d7ee40 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/C/a8e296c22c7e43ed8a55a72ca0d7ee40 2024-11-10T22:38:51,889 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/C/a8e296c22c7e43ed8a55a72ca0d7ee40, entries=150, sequenceid=242, filesize=11.9 K 2024-11-10T22:38:51,890 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 665ebb4a1f83b8f4b9dc2b71f1eaf80b in 1268ms, sequenceid=242, compaction requested=true 2024-11-10T22:38:51,890 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 665ebb4a1f83b8f4b9dc2b71f1eaf80b: 2024-11-10T22:38:51,890 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 665ebb4a1f83b8f4b9dc2b71f1eaf80b:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T22:38:51,890 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T22:38:51,890 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T22:38:51,890 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 665ebb4a1f83b8f4b9dc2b71f1eaf80b:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T22:38:51,890 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T22:38:51,890 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 665ebb4a1f83b8f4b9dc2b71f1eaf80b:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T22:38:51,890 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-10T22:38:51,891 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T22:38:51,891 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39287 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T22:38:51,891 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T22:38:51,892 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HStore(1541): 665ebb4a1f83b8f4b9dc2b71f1eaf80b/A is initiating minor compaction (all files) 2024-11-10T22:38:51,892 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1541): 665ebb4a1f83b8f4b9dc2b71f1eaf80b/B is initiating minor compaction (all files) 2024-11-10T22:38:51,892 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 665ebb4a1f83b8f4b9dc2b71f1eaf80b/B in TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b. 2024-11-10T22:38:51,892 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 665ebb4a1f83b8f4b9dc2b71f1eaf80b/A in TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b. 2024-11-10T22:38:51,892 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/A/aa73af33ebfc4e8591c8b6bf9568cac8, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/A/54dd7fea22e94151a760b8b635784d68, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/A/3c663e4a96e24a4cbb08724a4e3eaae8] into tmpdir=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp, totalSize=38.4 K 2024-11-10T22:38:51,892 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/B/0c741fe94efb424aa5ca9d5dfd76b3df, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/B/52fc2a922a0b48858a69e505949cd5b7, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/B/f9e6f5f6884d45aba56fcb86bacb8859] into tmpdir=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp, totalSize=36.0 K 2024-11-10T22:38:51,892 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting aa73af33ebfc4e8591c8b6bf9568cac8, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1731278327305 2024-11-10T22:38:51,892 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting 0c741fe94efb424aa5ca9d5dfd76b3df, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1731278327305 2024-11-10T22:38:51,893 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting 52fc2a922a0b48858a69e505949cd5b7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1731278328462 2024-11-10T22:38:51,893 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 54dd7fea22e94151a760b8b635784d68, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1731278328462 2024-11-10T22:38:51,893 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting f9e6f5f6884d45aba56fcb86bacb8859, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1731278330616 2024-11-10T22:38:51,893 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3c663e4a96e24a4cbb08724a4e3eaae8, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1731278330616 2024-11-10T22:38:51,904 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 665ebb4a1f83b8f4b9dc2b71f1eaf80b#B#compaction#269 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T22:38:51,904 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/B/5bd4b82172bf45d6bb7f557b9eead35b is 50, key is test_row_0/B:col10/1731278330616/Put/seqid=0 2024-11-10T22:38:51,911 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 665ebb4a1f83b8f4b9dc2b71f1eaf80b#A#compaction#270 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T22:38:51,912 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/A/915c3b12ad1f4491a2037702cf43be68 is 50, key is test_row_0/A:col10/1731278330616/Put/seqid=0 2024-11-10T22:38:51,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742144_1320 (size=12697) 2024-11-10T22:38:51,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742145_1321 (size=12697) 2024-11-10T22:38:51,944 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/B/5bd4b82172bf45d6bb7f557b9eead35b as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/B/5bd4b82172bf45d6bb7f557b9eead35b 2024-11-10T22:38:51,950 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/A/915c3b12ad1f4491a2037702cf43be68 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/A/915c3b12ad1f4491a2037702cf43be68 2024-11-10T22:38:51,956 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 665ebb4a1f83b8f4b9dc2b71f1eaf80b/B of 665ebb4a1f83b8f4b9dc2b71f1eaf80b into 5bd4b82172bf45d6bb7f557b9eead35b(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T22:38:51,956 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 665ebb4a1f83b8f4b9dc2b71f1eaf80b: 2024-11-10T22:38:51,956 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., storeName=665ebb4a1f83b8f4b9dc2b71f1eaf80b/B, priority=13, startTime=1731278331890; duration=0sec 2024-11-10T22:38:51,956 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T22:38:51,956 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 665ebb4a1f83b8f4b9dc2b71f1eaf80b:B 2024-11-10T22:38:51,956 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T22:38:51,958 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T22:38:51,958 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1541): 665ebb4a1f83b8f4b9dc2b71f1eaf80b/C is initiating minor compaction (all files) 2024-11-10T22:38:51,958 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 665ebb4a1f83b8f4b9dc2b71f1eaf80b/C in TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b. 2024-11-10T22:38:51,959 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/C/3c7ded45110944859af8bafe93df5cae, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/C/60115c271ed145a7a381e54f0b9da706, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/C/a8e296c22c7e43ed8a55a72ca0d7ee40] into tmpdir=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp, totalSize=36.0 K 2024-11-10T22:38:51,959 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 665ebb4a1f83b8f4b9dc2b71f1eaf80b/A of 665ebb4a1f83b8f4b9dc2b71f1eaf80b into 915c3b12ad1f4491a2037702cf43be68(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T22:38:51,959 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 665ebb4a1f83b8f4b9dc2b71f1eaf80b: 2024-11-10T22:38:51,959 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., storeName=665ebb4a1f83b8f4b9dc2b71f1eaf80b/A, priority=13, startTime=1731278331890; duration=0sec 2024-11-10T22:38:51,959 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T22:38:51,959 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting 3c7ded45110944859af8bafe93df5cae, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1731278327305 2024-11-10T22:38:51,959 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 665ebb4a1f83b8f4b9dc2b71f1eaf80b:A 2024-11-10T22:38:51,960 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting 60115c271ed145a7a381e54f0b9da706, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1731278328462 2024-11-10T22:38:51,960 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting a8e296c22c7e43ed8a55a72ca0d7ee40, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1731278330616 2024-11-10T22:38:51,966 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-10T22:38:51,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b. 2024-11-10T22:38:51,966 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2902): Flushing 665ebb4a1f83b8f4b9dc2b71f1eaf80b 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-10T22:38:51,982 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 665ebb4a1f83b8f4b9dc2b71f1eaf80b#C#compaction#271 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T22:38:51,983 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/C/842ee947976a414fbf62e3c9fae6cf92 is 50, key is test_row_0/C:col10/1731278330616/Put/seqid=0 2024-11-10T22:38:51,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/A/29dd34536b2f48e98771122c88dff420 is 50, key is test_row_0/A:col10/1731278330652/Put/seqid=0 2024-11-10T22:38:52,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742146_1322 (size=12301) 2024-11-10T22:38:52,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742147_1323 (size=12697) 2024-11-10T22:38:52,027 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/C/842ee947976a414fbf62e3c9fae6cf92 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/C/842ee947976a414fbf62e3c9fae6cf92 2024-11-10T22:38:52,034 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 665ebb4a1f83b8f4b9dc2b71f1eaf80b/C of 665ebb4a1f83b8f4b9dc2b71f1eaf80b into 842ee947976a414fbf62e3c9fae6cf92(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T22:38:52,034 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 665ebb4a1f83b8f4b9dc2b71f1eaf80b: 2024-11-10T22:38:52,034 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., storeName=665ebb4a1f83b8f4b9dc2b71f1eaf80b/C, priority=13, startTime=1731278331890; duration=0sec 2024-11-10T22:38:52,034 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T22:38:52,034 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 665ebb4a1f83b8f4b9dc2b71f1eaf80b:C 2024-11-10T22:38:52,406 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=269 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/A/29dd34536b2f48e98771122c88dff420 2024-11-10T22:38:52,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/B/f770e4eb49ff4f9a887a2151d7e81642 is 50, key is test_row_0/B:col10/1731278330652/Put/seqid=0 2024-11-10T22:38:52,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742148_1324 (size=12301) 2024-11-10T22:38:52,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-10T22:38:52,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(8855): Flush requested on 665ebb4a1f83b8f4b9dc2b71f1eaf80b 2024-11-10T22:38:52,804 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b. as already flushing 2024-11-10T22:38:52,818 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=269 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/B/f770e4eb49ff4f9a887a2151d7e81642 2024-11-10T22:38:52,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/C/e3a7883d66524b5ea79b1a7a16bbbd41 is 50, key is test_row_0/C:col10/1731278330652/Put/seqid=0 2024-11-10T22:38:52,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742149_1325 (size=12301) 2024-11-10T22:38:52,845 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:52,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46606 deadline: 1731278392839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:52,846 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:52,846 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:52,846 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:52,846 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:52,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46638 deadline: 1731278392841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:52,846 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:52,846 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:52,846 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:52,846 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:52,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1731278392841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:52,847 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:52,847 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:52,847 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:52,847 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:52,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46594 deadline: 1731278392842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:52,847 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:52,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46654 deadline: 1731278392842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:52,847 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:52,847 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:52,847 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:52,847 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:52,847 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:52,847 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:52,954 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:52,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46606 deadline: 1731278392953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:52,954 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:52,954 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:52,954 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:52,954 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:52,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46638 deadline: 1731278392953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:52,954 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:52,954 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:52,954 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:52,954 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:52,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1731278392953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:52,955 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:52,955 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:52,955 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:52,955 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:52,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46594 deadline: 1731278392953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:52,955 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:52,955 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:52,955 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:52,955 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:52,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46654 deadline: 1731278392953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:52,955 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:52,956 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:52,956 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:53,166 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:53,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46606 deadline: 1731278393163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:53,166 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:53,166 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:53,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46638 deadline: 1731278393163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:53,166 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:53,166 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:53,166 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:53,166 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:53,166 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:53,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1731278393163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:53,167 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:53,167 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:53,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46594 deadline: 1731278393163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:53,167 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:53,167 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:53,167 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:53,167 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:53,167 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:53,167 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:53,167 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:53,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46654 deadline: 1731278393163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:53,167 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:53,167 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:53,167 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:53,229 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=269 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/C/e3a7883d66524b5ea79b1a7a16bbbd41 2024-11-10T22:38:53,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/A/29dd34536b2f48e98771122c88dff420 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/A/29dd34536b2f48e98771122c88dff420 2024-11-10T22:38:53,238 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/A/29dd34536b2f48e98771122c88dff420, entries=150, sequenceid=269, filesize=12.0 K 2024-11-10T22:38:53,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/B/f770e4eb49ff4f9a887a2151d7e81642 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/B/f770e4eb49ff4f9a887a2151d7e81642 2024-11-10T22:38:53,242 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/B/f770e4eb49ff4f9a887a2151d7e81642, entries=150, sequenceid=269, filesize=12.0 K 2024-11-10T22:38:53,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/C/e3a7883d66524b5ea79b1a7a16bbbd41 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/C/e3a7883d66524b5ea79b1a7a16bbbd41 2024-11-10T22:38:53,247 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/C/e3a7883d66524b5ea79b1a7a16bbbd41, entries=150, sequenceid=269, filesize=12.0 K 2024-11-10T22:38:53,248 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=60.38 KB/61830 for 665ebb4a1f83b8f4b9dc2b71f1eaf80b in 1282ms, sequenceid=269, compaction requested=false 2024-11-10T22:38:53,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2603): Flush status journal for 665ebb4a1f83b8f4b9dc2b71f1eaf80b: 2024-11-10T22:38:53,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b. 2024-11-10T22:38:53,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=83 2024-11-10T22:38:53,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.HMaster(4169): Remote procedure done, pid=83 2024-11-10T22:38:53,251 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-11-10T22:38:53,251 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6620 sec 2024-11-10T22:38:53,253 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees in 2.6690 sec 2024-11-10T22:38:53,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(8855): Flush requested on 665ebb4a1f83b8f4b9dc2b71f1eaf80b 2024-11-10T22:38:53,475 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T22:38:53,476 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 665ebb4a1f83b8f4b9dc2b71f1eaf80b 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-10T22:38:53,481 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/A/1b1232e2b04d4b6c9e85a6d6f10df467 is 50, key is test_row_0/A:col10/1731278333476/Put/seqid=0 2024-11-10T22:38:53,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742150_1326 (size=12301) 2024-11-10T22:38:53,505 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:53,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46654 deadline: 1731278393499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:53,506 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:53,506 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:53,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46638 deadline: 1731278393500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:53,506 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:53,506 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:53,506 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:53,506 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:53,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46594 deadline: 1731278393501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:53,506 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:53,506 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:53,506 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:53,506 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:53,506 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:53,507 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:53,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46606 deadline: 1731278393503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:53,508 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:53,508 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:53,508 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:53,508 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:53,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1731278393503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:53,508 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:53,508 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:53,508 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:53,615 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:53,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46654 deadline: 1731278393612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:53,615 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:53,615 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:53,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46638 deadline: 1731278393612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:53,615 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:53,615 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:53,615 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:53,616 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:53,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46594 deadline: 1731278393612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:53,616 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:53,616 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:53,616 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:53,616 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:53,616 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:53,616 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:53,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46606 deadline: 1731278393612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:53,616 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:53,616 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:53,616 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:53,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1731278393612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:53,616 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:53,616 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:53,616 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:53,616 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:53,825 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:53,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46654 deadline: 1731278393822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:53,825 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:53,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46638 deadline: 1731278393822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:53,825 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:53,826 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:53,826 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:53,826 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:53,826 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:53,826 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:53,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46606 deadline: 1731278393822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:53,826 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:53,826 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:53,826 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:53,826 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:53,826 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:53,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46594 deadline: 1731278393822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:53,826 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:53,826 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:53,826 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:53,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1731278393822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:53,826 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:53,827 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:53,827 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:53,827 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:53,887 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=284 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/A/1b1232e2b04d4b6c9e85a6d6f10df467 2024-11-10T22:38:53,894 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/B/b5a46a25d7b74c6d873830749b481337 is 50, key is test_row_0/B:col10/1731278333476/Put/seqid=0 2024-11-10T22:38:53,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742151_1327 (size=12301) 2024-11-10T22:38:54,135 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:54,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46654 deadline: 1731278394132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:54,136 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:54,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46638 deadline: 1731278394132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:54,136 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:54,136 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:54,136 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:54,136 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:54,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46606 deadline: 1731278394132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:54,136 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:54,136 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:54,136 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:54,136 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:54,136 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:54,136 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:54,136 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:54,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46594 deadline: 1731278394132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:54,137 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:54,137 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:54,137 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:54,137 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:54,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1731278394132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:54,137 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:54,137 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:54,137 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:54,299 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=284 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/B/b5a46a25d7b74c6d873830749b481337 2024-11-10T22:38:54,306 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/C/d958a5f5f88848c2a17368120136ad0e is 50, key is test_row_0/C:col10/1731278333476/Put/seqid=0 2024-11-10T22:38:54,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742152_1328 (size=12301) 2024-11-10T22:38:54,645 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:54,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46654 deadline: 1731278394642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:54,646 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:54,646 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:54,646 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:54,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46638 deadline: 1731278394642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:54,646 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:54,646 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:54,646 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:54,646 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:54,646 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:54,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46606 deadline: 1731278394642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:54,646 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:54,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46594 deadline: 1731278394642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:54,647 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:54,647 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:54,647 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:54,647 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:54,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1731278394643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:54,647 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:54,647 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:54,647 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:54,647 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:54,647 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:54,647 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:54,710 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=284 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/C/d958a5f5f88848c2a17368120136ad0e 2024-11-10T22:38:54,715 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/A/1b1232e2b04d4b6c9e85a6d6f10df467 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/A/1b1232e2b04d4b6c9e85a6d6f10df467 2024-11-10T22:38:54,719 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/A/1b1232e2b04d4b6c9e85a6d6f10df467, entries=150, sequenceid=284, filesize=12.0 K 2024-11-10T22:38:54,720 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/B/b5a46a25d7b74c6d873830749b481337 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/B/b5a46a25d7b74c6d873830749b481337 2024-11-10T22:38:54,723 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/B/b5a46a25d7b74c6d873830749b481337, entries=150, sequenceid=284, filesize=12.0 K 2024-11-10T22:38:54,724 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/C/d958a5f5f88848c2a17368120136ad0e as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/C/d958a5f5f88848c2a17368120136ad0e 2024-11-10T22:38:54,728 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/C/d958a5f5f88848c2a17368120136ad0e, entries=150, sequenceid=284, filesize=12.0 K 2024-11-10T22:38:54,729 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=134.18 KB/137400 for 665ebb4a1f83b8f4b9dc2b71f1eaf80b in 1253ms, sequenceid=284, compaction requested=true 2024-11-10T22:38:54,729 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 665ebb4a1f83b8f4b9dc2b71f1eaf80b: 2024-11-10T22:38:54,729 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 665ebb4a1f83b8f4b9dc2b71f1eaf80b:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T22:38:54,729 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T22:38:54,729 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T22:38:54,729 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 665ebb4a1f83b8f4b9dc2b71f1eaf80b:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T22:38:54,729 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T22:38:54,729 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 665ebb4a1f83b8f4b9dc2b71f1eaf80b:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T22:38:54,729 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T22:38:54,729 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T22:38:54,730 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37299 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T22:38:54,730 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1541): 665ebb4a1f83b8f4b9dc2b71f1eaf80b/B is initiating minor compaction (all files) 2024-11-10T22:38:54,730 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 665ebb4a1f83b8f4b9dc2b71f1eaf80b/B in TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b. 2024-11-10T22:38:54,730 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/B/5bd4b82172bf45d6bb7f557b9eead35b, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/B/f770e4eb49ff4f9a887a2151d7e81642, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/B/b5a46a25d7b74c6d873830749b481337] into tmpdir=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp, totalSize=36.4 K 2024-11-10T22:38:54,730 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37299 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T22:38:54,730 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HStore(1541): 665ebb4a1f83b8f4b9dc2b71f1eaf80b/A is initiating minor compaction (all files) 2024-11-10T22:38:54,730 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 665ebb4a1f83b8f4b9dc2b71f1eaf80b/A in TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b. 2024-11-10T22:38:54,730 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/A/915c3b12ad1f4491a2037702cf43be68, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/A/29dd34536b2f48e98771122c88dff420, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/A/1b1232e2b04d4b6c9e85a6d6f10df467] into tmpdir=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp, totalSize=36.4 K 2024-11-10T22:38:54,731 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 915c3b12ad1f4491a2037702cf43be68, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1731278330616 2024-11-10T22:38:54,731 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting 5bd4b82172bf45d6bb7f557b9eead35b, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1731278330616 2024-11-10T22:38:54,731 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 29dd34536b2f48e98771122c88dff420, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=269, earliestPutTs=1731278330652 2024-11-10T22:38:54,731 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting f770e4eb49ff4f9a887a2151d7e81642, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=269, earliestPutTs=1731278330652 2024-11-10T22:38:54,731 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1b1232e2b04d4b6c9e85a6d6f10df467, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1731278332840 2024-11-10T22:38:54,731 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting b5a46a25d7b74c6d873830749b481337, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1731278332840 2024-11-10T22:38:54,741 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 665ebb4a1f83b8f4b9dc2b71f1eaf80b#A#compaction#278 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T22:38:54,742 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/A/c12930ee69934102b0ebdbfcfc18dd76 is 50, key is test_row_0/A:col10/1731278333476/Put/seqid=0 2024-11-10T22:38:54,744 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 665ebb4a1f83b8f4b9dc2b71f1eaf80b#B#compaction#279 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T22:38:54,744 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/B/4e6ae51617ba4aeeadc941548d8f68bb is 50, key is test_row_0/B:col10/1731278333476/Put/seqid=0 2024-11-10T22:38:54,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742153_1329 (size=12949) 2024-11-10T22:38:54,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-10T22:38:54,763 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T22:38:54,764 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T22:38:54,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] procedure2.ProcedureExecutor(1139): Stored pid=84, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees 2024-11-10T22:38:54,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-10T22:38:54,766 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/A/c12930ee69934102b0ebdbfcfc18dd76 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/A/c12930ee69934102b0ebdbfcfc18dd76 2024-11-10T22:38:54,767 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=84, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T22:38:54,768 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=84, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T22:38:54,768 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T22:38:54,774 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 665ebb4a1f83b8f4b9dc2b71f1eaf80b/A of 665ebb4a1f83b8f4b9dc2b71f1eaf80b into c12930ee69934102b0ebdbfcfc18dd76(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T22:38:54,774 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 665ebb4a1f83b8f4b9dc2b71f1eaf80b: 2024-11-10T22:38:54,774 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., storeName=665ebb4a1f83b8f4b9dc2b71f1eaf80b/A, priority=13, startTime=1731278334729; duration=0sec 2024-11-10T22:38:54,774 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T22:38:54,774 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 665ebb4a1f83b8f4b9dc2b71f1eaf80b:A 2024-11-10T22:38:54,774 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T22:38:54,776 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37299 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T22:38:54,776 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HStore(1541): 665ebb4a1f83b8f4b9dc2b71f1eaf80b/C is initiating minor compaction (all files) 2024-11-10T22:38:54,776 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742154_1330 (size=12949) 2024-11-10T22:38:54,776 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 665ebb4a1f83b8f4b9dc2b71f1eaf80b/C in TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b. 2024-11-10T22:38:54,777 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/C/842ee947976a414fbf62e3c9fae6cf92, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/C/e3a7883d66524b5ea79b1a7a16bbbd41, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/C/d958a5f5f88848c2a17368120136ad0e] into tmpdir=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp, totalSize=36.4 K 2024-11-10T22:38:54,777 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 842ee947976a414fbf62e3c9fae6cf92, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1731278330616 2024-11-10T22:38:54,777 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting e3a7883d66524b5ea79b1a7a16bbbd41, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=269, earliestPutTs=1731278330652 2024-11-10T22:38:54,778 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting d958a5f5f88848c2a17368120136ad0e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1731278332840 2024-11-10T22:38:54,782 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/B/4e6ae51617ba4aeeadc941548d8f68bb as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/B/4e6ae51617ba4aeeadc941548d8f68bb 2024-11-10T22:38:54,787 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 665ebb4a1f83b8f4b9dc2b71f1eaf80b#C#compaction#280 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T22:38:54,788 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/C/6f49cdc405004f96b5e6ee3a7c8d8b49 is 50, key is test_row_0/C:col10/1731278333476/Put/seqid=0 2024-11-10T22:38:54,788 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 665ebb4a1f83b8f4b9dc2b71f1eaf80b/B of 665ebb4a1f83b8f4b9dc2b71f1eaf80b into 4e6ae51617ba4aeeadc941548d8f68bb(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T22:38:54,788 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 665ebb4a1f83b8f4b9dc2b71f1eaf80b: 2024-11-10T22:38:54,788 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., storeName=665ebb4a1f83b8f4b9dc2b71f1eaf80b/B, priority=13, startTime=1731278334729; duration=0sec 2024-11-10T22:38:54,788 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T22:38:54,788 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 665ebb4a1f83b8f4b9dc2b71f1eaf80b:B 2024-11-10T22:38:54,792 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742155_1331 (size=12949) 2024-11-10T22:38:54,800 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/C/6f49cdc405004f96b5e6ee3a7c8d8b49 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/C/6f49cdc405004f96b5e6ee3a7c8d8b49 2024-11-10T22:38:54,806 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 665ebb4a1f83b8f4b9dc2b71f1eaf80b/C of 665ebb4a1f83b8f4b9dc2b71f1eaf80b into 6f49cdc405004f96b5e6ee3a7c8d8b49(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T22:38:54,806 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 665ebb4a1f83b8f4b9dc2b71f1eaf80b: 2024-11-10T22:38:54,806 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., storeName=665ebb4a1f83b8f4b9dc2b71f1eaf80b/C, priority=13, startTime=1731278334729; duration=0sec 2024-11-10T22:38:54,806 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T22:38:54,806 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 665ebb4a1f83b8f4b9dc2b71f1eaf80b:C 2024-11-10T22:38:54,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-10T22:38:54,921 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-10T22:38:54,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b. 2024-11-10T22:38:54,922 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2902): Flushing 665ebb4a1f83b8f4b9dc2b71f1eaf80b 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-10T22:38:54,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/A/2a0a9ce8ffb844f08981933b89487713 is 50, key is test_row_0/A:col10/1731278333502/Put/seqid=0 2024-11-10T22:38:54,931 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742156_1332 (size=12301) 2024-11-10T22:38:54,933 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=310 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/A/2a0a9ce8ffb844f08981933b89487713 2024-11-10T22:38:54,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/B/e1ac1973bfec48898cecda14a3e2e13a is 50, key is test_row_0/B:col10/1731278333502/Put/seqid=0 2024-11-10T22:38:54,944 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742157_1333 (size=12301) 2024-11-10T22:38:54,944 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=310 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/B/e1ac1973bfec48898cecda14a3e2e13a 2024-11-10T22:38:54,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/C/b84d4dbf2ac34d5d8b275283f8c8a8de is 50, key is test_row_0/C:col10/1731278333502/Put/seqid=0 2024-11-10T22:38:54,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742158_1334 (size=12301) 2024-11-10T22:38:54,956 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=310 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/C/b84d4dbf2ac34d5d8b275283f8c8a8de 2024-11-10T22:38:54,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/A/2a0a9ce8ffb844f08981933b89487713 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/A/2a0a9ce8ffb844f08981933b89487713 2024-11-10T22:38:54,965 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/A/2a0a9ce8ffb844f08981933b89487713, entries=150, sequenceid=310, filesize=12.0 K 2024-11-10T22:38:54,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/B/e1ac1973bfec48898cecda14a3e2e13a as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/B/e1ac1973bfec48898cecda14a3e2e13a 2024-11-10T22:38:54,983 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/B/e1ac1973bfec48898cecda14a3e2e13a, entries=150, sequenceid=310, filesize=12.0 K 2024-11-10T22:38:54,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/C/b84d4dbf2ac34d5d8b275283f8c8a8de as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/C/b84d4dbf2ac34d5d8b275283f8c8a8de 2024-11-10T22:38:54,990 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/C/b84d4dbf2ac34d5d8b275283f8c8a8de, entries=150, sequenceid=310, filesize=12.0 K 2024-11-10T22:38:54,991 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=0 B/0 for 665ebb4a1f83b8f4b9dc2b71f1eaf80b in 69ms, sequenceid=310, compaction requested=false 2024-11-10T22:38:54,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2603): Flush status journal for 665ebb4a1f83b8f4b9dc2b71f1eaf80b: 2024-11-10T22:38:54,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b. 2024-11-10T22:38:54,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=85 2024-11-10T22:38:54,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.HMaster(4169): Remote procedure done, pid=85 2024-11-10T22:38:54,996 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-11-10T22:38:54,996 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 225 msec 2024-11-10T22:38:54,999 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees in 233 msec 2024-11-10T22:38:55,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-10T22:38:55,093 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T22:38:55,093 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T22:38:55,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] procedure2.ProcedureExecutor(1139): Stored pid=86, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees 2024-11-10T22:38:55,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-10T22:38:55,095 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=86, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T22:38:55,096 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=86, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T22:38:55,096 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=87, ppid=86, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T22:38:55,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-10T22:38:55,249 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-10T22:38:55,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b. 2024-11-10T22:38:55,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2603): Flush status journal for 665ebb4a1f83b8f4b9dc2b71f1eaf80b: 2024-11-10T22:38:55,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b. 2024-11-10T22:38:55,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=87 2024-11-10T22:38:55,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.HMaster(4169): Remote procedure done, pid=87 2024-11-10T22:38:55,253 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=87, resume processing ppid=86 2024-11-10T22:38:55,253 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, ppid=86, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 155 msec 2024-11-10T22:38:55,256 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees in 161 msec 2024-11-10T22:38:55,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-10T22:38:55,412 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T22:38:55,413 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T22:38:55,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] procedure2.ProcedureExecutor(1139): Stored pid=88, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=88, table=TestAcidGuarantees 2024-11-10T22:38:55,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-10T22:38:55,415 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=88, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=88, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T22:38:55,416 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=88, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=88, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T22:38:55,416 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=89, ppid=88, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T22:38:55,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-10T22:38:55,568 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=89 2024-11-10T22:38:55,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b. 2024-11-10T22:38:55,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegion(2603): Flush status journal for 665ebb4a1f83b8f4b9dc2b71f1eaf80b: 2024-11-10T22:38:55,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b. 2024-11-10T22:38:55,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=89 2024-11-10T22:38:55,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.HMaster(4169): Remote procedure done, pid=89 2024-11-10T22:38:55,572 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=89, resume processing ppid=88 2024-11-10T22:38:55,572 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, ppid=88, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 154 msec 2024-11-10T22:38:55,574 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=88, table=TestAcidGuarantees in 160 msec 2024-11-10T22:38:55,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(8855): Flush requested on 665ebb4a1f83b8f4b9dc2b71f1eaf80b 2024-11-10T22:38:55,662 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T22:38:55,662 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 665ebb4a1f83b8f4b9dc2b71f1eaf80b 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-10T22:38:55,667 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/A/510e129d99a44fe9a46d0cd7d19bec6f is 50, key is test_row_0/A:col10/1731278335655/Put/seqid=0 2024-11-10T22:38:55,672 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742159_1335 (size=14741) 2024-11-10T22:38:55,692 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:55,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1731278395685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:55,693 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:55,693 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:55,693 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:55,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46654 deadline: 1731278395686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:55,693 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:55,693 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:55,693 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:55,693 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:55,693 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:55,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46638 deadline: 1731278395686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:55,693 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:55,693 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:55,693 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:55,694 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:55,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46606 deadline: 1731278395688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:55,694 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:55,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46594 deadline: 1731278395689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:55,694 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:55,694 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:55,694 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:55,694 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:55,694 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:55,694 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:55,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-10T22:38:55,733 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T22:38:55,733 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T22:38:55,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] procedure2.ProcedureExecutor(1139): Stored pid=90, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=90, table=TestAcidGuarantees 2024-11-10T22:38:55,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-10T22:38:55,735 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=90, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=90, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T22:38:55,736 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=90, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=90, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T22:38:55,736 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=91, ppid=90, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T22:38:55,805 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:55,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1731278395802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:55,806 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:55,806 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:55,806 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:55,806 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:55,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46654 deadline: 1731278395802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:55,806 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:55,806 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:55,806 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:55,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46638 deadline: 1731278395802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:55,806 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:55,806 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:55,807 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:55,807 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:55,807 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:55,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46606 deadline: 1731278395802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:55,807 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:55,807 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:55,807 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:55,807 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:55,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46594 deadline: 1731278395802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:55,807 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:55,807 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:55,807 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:55,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-10T22:38:55,888 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=91 2024-11-10T22:38:55,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b. 2024-11-10T22:38:55,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b. as already flushing 2024-11-10T22:38:55,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b. 2024-11-10T22:38:55,888 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] handler.RSProcedureHandler(58): pid=91 java.io.IOException: Unable to complete flush {ENCODED => 665ebb4a1f83b8f4b9dc2b71f1eaf80b, NAME => 'TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:55,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=91 java.io.IOException: Unable to complete flush {ENCODED => 665ebb4a1f83b8f4b9dc2b71f1eaf80b, NAME => 'TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:55,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=91 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 665ebb4a1f83b8f4b9dc2b71f1eaf80b, NAME => 'TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 665ebb4a1f83b8f4b9dc2b71f1eaf80b, NAME => 'TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:56,016 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:56,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1731278396012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:56,016 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:56,016 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:56,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46654 deadline: 1731278396012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:56,017 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:56,017 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:56,017 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:56,017 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:56,017 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:56,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46638 deadline: 1731278396013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:56,017 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:56,017 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:56,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46606 deadline: 1731278396013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:56,017 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:56,017 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:56,017 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:56,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46594 deadline: 1731278396013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:56,017 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:56,017 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:56,017 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:56,017 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:56,018 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:56,018 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:56,018 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:56,041 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=91 2024-11-10T22:38:56,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b. 2024-11-10T22:38:56,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b. as already flushing 2024-11-10T22:38:56,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b. 2024-11-10T22:38:56,041 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] handler.RSProcedureHandler(58): pid=91 java.io.IOException: Unable to complete flush {ENCODED => 665ebb4a1f83b8f4b9dc2b71f1eaf80b, NAME => 'TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:56,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=91 java.io.IOException: Unable to complete flush {ENCODED => 665ebb4a1f83b8f4b9dc2b71f1eaf80b, NAME => 'TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:56,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=91 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 665ebb4a1f83b8f4b9dc2b71f1eaf80b, NAME => 'TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 665ebb4a1f83b8f4b9dc2b71f1eaf80b, NAME => 'TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:56,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-10T22:38:56,073 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=323 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/A/510e129d99a44fe9a46d0cd7d19bec6f 2024-11-10T22:38:56,080 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/B/bac4d3c649654f7fa4e73f8345fc4ea4 is 50, key is test_row_0/B:col10/1731278335655/Put/seqid=0 2024-11-10T22:38:56,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742160_1336 (size=12301) 2024-11-10T22:38:56,194 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=91 2024-11-10T22:38:56,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b. 2024-11-10T22:38:56,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b. as already flushing 2024-11-10T22:38:56,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b. 2024-11-10T22:38:56,194 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] handler.RSProcedureHandler(58): pid=91 java.io.IOException: Unable to complete flush {ENCODED => 665ebb4a1f83b8f4b9dc2b71f1eaf80b, NAME => 'TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:56,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=91 java.io.IOException: Unable to complete flush {ENCODED => 665ebb4a1f83b8f4b9dc2b71f1eaf80b, NAME => 'TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:56,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=91 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 665ebb4a1f83b8f4b9dc2b71f1eaf80b, NAME => 'TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 665ebb4a1f83b8f4b9dc2b71f1eaf80b, NAME => 'TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:56,326 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:56,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1731278396322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:56,326 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:56,326 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:56,326 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:56,327 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:56,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46654 deadline: 1731278396322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:56,327 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:56,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46638 deadline: 1731278396322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:56,327 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:56,327 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:56,327 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:56,327 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:56,327 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:56,327 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:56,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46606 deadline: 1731278396323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:56,327 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:56,327 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:56,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46594 deadline: 1731278396323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:56,327 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:56,327 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:56,327 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:56,327 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:56,327 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:56,327 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:56,347 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=91 2024-11-10T22:38:56,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b. 2024-11-10T22:38:56,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b. as already flushing 2024-11-10T22:38:56,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b. 2024-11-10T22:38:56,347 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] handler.RSProcedureHandler(58): pid=91 java.io.IOException: Unable to complete flush {ENCODED => 665ebb4a1f83b8f4b9dc2b71f1eaf80b, NAME => 'TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:56,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=91 java.io.IOException: Unable to complete flush {ENCODED => 665ebb4a1f83b8f4b9dc2b71f1eaf80b, NAME => 'TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:56,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=91 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 665ebb4a1f83b8f4b9dc2b71f1eaf80b, NAME => 'TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 665ebb4a1f83b8f4b9dc2b71f1eaf80b, NAME => 'TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:56,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-10T22:38:56,484 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=323 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/B/bac4d3c649654f7fa4e73f8345fc4ea4 2024-11-10T22:38:56,492 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/C/21c19a5edf1f46f9bd3ae2383b7c36c4 is 50, key is test_row_0/C:col10/1731278335655/Put/seqid=0 2024-11-10T22:38:56,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742161_1337 (size=12301) 2024-11-10T22:38:56,500 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=91 2024-11-10T22:38:56,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b. 2024-11-10T22:38:56,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b. as already flushing 2024-11-10T22:38:56,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b. 2024-11-10T22:38:56,500 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] handler.RSProcedureHandler(58): pid=91 java.io.IOException: Unable to complete flush {ENCODED => 665ebb4a1f83b8f4b9dc2b71f1eaf80b, NAME => 'TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:56,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=91 java.io.IOException: Unable to complete flush {ENCODED => 665ebb4a1f83b8f4b9dc2b71f1eaf80b, NAME => 'TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:56,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=91 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 665ebb4a1f83b8f4b9dc2b71f1eaf80b, NAME => 'TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 665ebb4a1f83b8f4b9dc2b71f1eaf80b, NAME => 'TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:56,653 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=91 2024-11-10T22:38:56,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b. 2024-11-10T22:38:56,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b. as already flushing 2024-11-10T22:38:56,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b. 2024-11-10T22:38:56,653 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] handler.RSProcedureHandler(58): pid=91 java.io.IOException: Unable to complete flush {ENCODED => 665ebb4a1f83b8f4b9dc2b71f1eaf80b, NAME => 'TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:56,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=91 java.io.IOException: Unable to complete flush {ENCODED => 665ebb4a1f83b8f4b9dc2b71f1eaf80b, NAME => 'TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:56,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=91 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 665ebb4a1f83b8f4b9dc2b71f1eaf80b, NAME => 'TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 665ebb4a1f83b8f4b9dc2b71f1eaf80b, NAME => 'TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:56,806 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=91 2024-11-10T22:38:56,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b. 2024-11-10T22:38:56,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b. as already flushing 2024-11-10T22:38:56,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b. 2024-11-10T22:38:56,806 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] handler.RSProcedureHandler(58): pid=91 java.io.IOException: Unable to complete flush {ENCODED => 665ebb4a1f83b8f4b9dc2b71f1eaf80b, NAME => 'TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:56,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=91 java.io.IOException: Unable to complete flush {ENCODED => 665ebb4a1f83b8f4b9dc2b71f1eaf80b, NAME => 'TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:56,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=91 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 665ebb4a1f83b8f4b9dc2b71f1eaf80b, NAME => 'TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 665ebb4a1f83b8f4b9dc2b71f1eaf80b, NAME => 'TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:38:56,834 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:56,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46632 deadline: 1731278396833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:56,835 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:56,835 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:56,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46638 deadline: 1731278396833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:56,835 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:56,835 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:56,835 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:56,835 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:56,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46606 deadline: 1731278396833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:56,835 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:56,835 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:56,836 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:56,836 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:56,836 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:56,836 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:56,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46594 deadline: 1731278396833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:56,836 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:56,836 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:56,836 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:56,845 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:38:56,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46654 deadline: 1731278396842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 2024-11-10T22:38:56,846 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:56,846 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:38:56,846 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:38:56,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-10T22:38:56,897 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=323 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/C/21c19a5edf1f46f9bd3ae2383b7c36c4 2024-11-10T22:38:56,902 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/A/510e129d99a44fe9a46d0cd7d19bec6f as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/A/510e129d99a44fe9a46d0cd7d19bec6f 2024-11-10T22:38:56,906 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/A/510e129d99a44fe9a46d0cd7d19bec6f, entries=200, sequenceid=323, filesize=14.4 K 2024-11-10T22:38:56,907 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/B/bac4d3c649654f7fa4e73f8345fc4ea4 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/B/bac4d3c649654f7fa4e73f8345fc4ea4 2024-11-10T22:38:56,911 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/B/bac4d3c649654f7fa4e73f8345fc4ea4, entries=150, sequenceid=323, filesize=12.0 K 2024-11-10T22:38:56,912 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/C/21c19a5edf1f46f9bd3ae2383b7c36c4 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/C/21c19a5edf1f46f9bd3ae2383b7c36c4 2024-11-10T22:38:56,916 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/C/21c19a5edf1f46f9bd3ae2383b7c36c4, entries=150, sequenceid=323, filesize=12.0 K 2024-11-10T22:38:56,916 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 665ebb4a1f83b8f4b9dc2b71f1eaf80b in 1254ms, sequenceid=323, compaction requested=true 2024-11-10T22:38:56,917 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 665ebb4a1f83b8f4b9dc2b71f1eaf80b: 2024-11-10T22:38:56,917 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 665ebb4a1f83b8f4b9dc2b71f1eaf80b:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T22:38:56,917 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T22:38:56,917 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 665ebb4a1f83b8f4b9dc2b71f1eaf80b:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T22:38:56,917 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T22:38:56,917 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 665ebb4a1f83b8f4b9dc2b71f1eaf80b:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T22:38:56,917 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T22:38:56,917 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T22:38:56,917 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T22:38:56,918 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T22:38:56,918 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39991 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T22:38:56,918 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1541): 665ebb4a1f83b8f4b9dc2b71f1eaf80b/B is initiating minor compaction (all files) 2024-11-10T22:38:56,918 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HStore(1541): 665ebb4a1f83b8f4b9dc2b71f1eaf80b/A is initiating minor compaction (all files) 2024-11-10T22:38:56,918 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 665ebb4a1f83b8f4b9dc2b71f1eaf80b/B in TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b. 2024-11-10T22:38:56,918 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 665ebb4a1f83b8f4b9dc2b71f1eaf80b/A in TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b. 2024-11-10T22:38:56,918 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/B/4e6ae51617ba4aeeadc941548d8f68bb, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/B/e1ac1973bfec48898cecda14a3e2e13a, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/B/bac4d3c649654f7fa4e73f8345fc4ea4] into tmpdir=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp, totalSize=36.7 K 2024-11-10T22:38:56,918 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/A/c12930ee69934102b0ebdbfcfc18dd76, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/A/2a0a9ce8ffb844f08981933b89487713, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/A/510e129d99a44fe9a46d0cd7d19bec6f] into tmpdir=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp, totalSize=39.1 K 2024-11-10T22:38:56,918 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting 4e6ae51617ba4aeeadc941548d8f68bb, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1731278332840 2024-11-10T22:38:56,918 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting c12930ee69934102b0ebdbfcfc18dd76, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1731278332840 2024-11-10T22:38:56,919 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting e1ac1973bfec48898cecda14a3e2e13a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1731278333498 2024-11-10T22:38:56,919 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2a0a9ce8ffb844f08981933b89487713, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1731278333498 2024-11-10T22:38:56,919 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting bac4d3c649654f7fa4e73f8345fc4ea4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1731278335655 2024-11-10T22:38:56,920 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 510e129d99a44fe9a46d0cd7d19bec6f, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1731278335655 2024-11-10T22:38:56,927 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 665ebb4a1f83b8f4b9dc2b71f1eaf80b#B#compaction#287 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T22:38:56,927 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/B/c4d43dbfe382493fb071bc89944652a8 is 50, key is test_row_0/B:col10/1731278335655/Put/seqid=0 2024-11-10T22:38:56,930 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 665ebb4a1f83b8f4b9dc2b71f1eaf80b#A#compaction#288 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T22:38:56,931 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/A/339e92f827cd45ab8ad8d0a6a07764e0 is 50, key is test_row_0/A:col10/1731278335655/Put/seqid=0 2024-11-10T22:38:56,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742162_1338 (size=13051) 2024-11-10T22:38:56,946 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/B/c4d43dbfe382493fb071bc89944652a8 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/B/c4d43dbfe382493fb071bc89944652a8 2024-11-10T22:38:56,951 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 665ebb4a1f83b8f4b9dc2b71f1eaf80b/B of 665ebb4a1f83b8f4b9dc2b71f1eaf80b into c4d43dbfe382493fb071bc89944652a8(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T22:38:56,951 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 665ebb4a1f83b8f4b9dc2b71f1eaf80b: 2024-11-10T22:38:56,951 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., storeName=665ebb4a1f83b8f4b9dc2b71f1eaf80b/B, priority=13, startTime=1731278336917; duration=0sec 2024-11-10T22:38:56,951 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T22:38:56,951 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 665ebb4a1f83b8f4b9dc2b71f1eaf80b:B 2024-11-10T22:38:56,951 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T22:38:56,952 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T22:38:56,952 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1541): 665ebb4a1f83b8f4b9dc2b71f1eaf80b/C is initiating minor compaction (all files) 2024-11-10T22:38:56,952 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 665ebb4a1f83b8f4b9dc2b71f1eaf80b/C in TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b. 2024-11-10T22:38:56,952 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/C/6f49cdc405004f96b5e6ee3a7c8d8b49, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/C/b84d4dbf2ac34d5d8b275283f8c8a8de, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/C/21c19a5edf1f46f9bd3ae2383b7c36c4] into tmpdir=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp, totalSize=36.7 K 2024-11-10T22:38:56,952 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting 6f49cdc405004f96b5e6ee3a7c8d8b49, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1731278332840 2024-11-10T22:38:56,953 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting b84d4dbf2ac34d5d8b275283f8c8a8de, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1731278333498 2024-11-10T22:38:56,953 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting 21c19a5edf1f46f9bd3ae2383b7c36c4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1731278335655 2024-11-10T22:38:56,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742163_1339 (size=13051) 2024-11-10T22:38:56,960 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=91 2024-11-10T22:38:56,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b. 2024-11-10T22:38:56,960 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 665ebb4a1f83b8f4b9dc2b71f1eaf80b#C#compaction#289 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T22:38:56,960 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegion(2902): Flushing 665ebb4a1f83b8f4b9dc2b71f1eaf80b 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-10T22:38:56,961 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/C/59ddcbb53efb4cac859c1a56dbde1325 is 50, key is test_row_0/C:col10/1731278335655/Put/seqid=0 2024-11-10T22:38:56,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/A/66149c004a4b4fb5b41870a0dca038ba is 50, key is test_row_0/A:col10/1731278335682/Put/seqid=0 2024-11-10T22:38:56,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742164_1340 (size=13051) 2024-11-10T22:38:56,984 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/C/59ddcbb53efb4cac859c1a56dbde1325 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/C/59ddcbb53efb4cac859c1a56dbde1325 2024-11-10T22:38:56,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742165_1341 (size=12301) 2024-11-10T22:38:56,987 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=349 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/A/66149c004a4b4fb5b41870a0dca038ba 2024-11-10T22:38:56,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/B/17f15a03098045b5b9c82dc9c8a7abac is 50, key is test_row_0/B:col10/1731278335682/Put/seqid=0 2024-11-10T22:38:56,995 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 665ebb4a1f83b8f4b9dc2b71f1eaf80b/C of 665ebb4a1f83b8f4b9dc2b71f1eaf80b into 59ddcbb53efb4cac859c1a56dbde1325(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T22:38:56,995 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 665ebb4a1f83b8f4b9dc2b71f1eaf80b: 2024-11-10T22:38:56,995 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., storeName=665ebb4a1f83b8f4b9dc2b71f1eaf80b/C, priority=13, startTime=1731278336917; duration=0sec 2024-11-10T22:38:56,995 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T22:38:56,996 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 665ebb4a1f83b8f4b9dc2b71f1eaf80b:C 2024-11-10T22:38:56,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742166_1342 (size=12301) 2024-11-10T22:38:56,998 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=349 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/B/17f15a03098045b5b9c82dc9c8a7abac 2024-11-10T22:38:57,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/C/aa2bc932a88e42aa9c53d8ec66f05c00 is 50, key is test_row_0/C:col10/1731278335682/Put/seqid=0 2024-11-10T22:38:57,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742167_1343 (size=12301) 2024-11-10T22:38:57,014 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=349 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/C/aa2bc932a88e42aa9c53d8ec66f05c00 2024-11-10T22:38:57,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/A/66149c004a4b4fb5b41870a0dca038ba as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/A/66149c004a4b4fb5b41870a0dca038ba 2024-11-10T22:38:57,023 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/A/66149c004a4b4fb5b41870a0dca038ba, entries=150, sequenceid=349, filesize=12.0 K 2024-11-10T22:38:57,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/B/17f15a03098045b5b9c82dc9c8a7abac as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/B/17f15a03098045b5b9c82dc9c8a7abac 2024-11-10T22:38:57,028 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/B/17f15a03098045b5b9c82dc9c8a7abac, entries=150, sequenceid=349, filesize=12.0 K 2024-11-10T22:38:57,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/C/aa2bc932a88e42aa9c53d8ec66f05c00 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/C/aa2bc932a88e42aa9c53d8ec66f05c00 2024-11-10T22:38:57,032 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/C/aa2bc932a88e42aa9c53d8ec66f05c00, entries=150, sequenceid=349, filesize=12.0 K 2024-11-10T22:38:57,033 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=0 B/0 for 665ebb4a1f83b8f4b9dc2b71f1eaf80b in 73ms, sequenceid=349, compaction requested=false 2024-11-10T22:38:57,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegion(2603): Flush status journal for 665ebb4a1f83b8f4b9dc2b71f1eaf80b: 2024-11-10T22:38:57,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b. 2024-11-10T22:38:57,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=91 2024-11-10T22:38:57,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.HMaster(4169): Remote procedure done, pid=91 2024-11-10T22:38:57,037 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=91, resume processing ppid=90 2024-11-10T22:38:57,037 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, ppid=90, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2980 sec 2024-11-10T22:38:57,039 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=90, table=TestAcidGuarantees in 1.3050 sec 2024-11-10T22:38:57,222 INFO [Thread-1261 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1261. 2024-11-10T22:38:57,222 DEBUG [Thread-1261 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T22:38:57,222 INFO [Thread-1259 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1259. 2024-11-10T22:38:57,222 DEBUG [Thread-1261 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:38:57,222 DEBUG [Thread-1259 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T22:38:57,222 DEBUG [Thread-1259 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:38:57,222 DEBUG [Thread-1259 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:38:57,222 DEBUG [Thread-1261 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:38:57,222 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T22:38:57,222 INFO [Thread-1263 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1263. 2024-11-10T22:38:57,222 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T22:38:57,222 DEBUG [Thread-1263 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T22:38:57,222 DEBUG [Thread-1263 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:38:57,223 INFO [Thread-1255 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1255. 2024-11-10T22:38:57,223 DEBUG [Thread-1263 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:38:57,223 DEBUG [Thread-1255 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T22:38:57,223 DEBUG [Thread-1255 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:38:57,223 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T22:38:57,223 DEBUG [Thread-1255 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:38:57,223 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T22:38:57,223 INFO [Thread-1257 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1257. 2024-11-10T22:38:57,223 DEBUG [Thread-1257 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T22:38:57,223 DEBUG [Thread-1257 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:38:57,223 DEBUG [Thread-1257 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:38:57,223 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T22:38:57,365 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/A/339e92f827cd45ab8ad8d0a6a07764e0 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/A/339e92f827cd45ab8ad8d0a6a07764e0 2024-11-10T22:38:57,369 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 665ebb4a1f83b8f4b9dc2b71f1eaf80b/A of 665ebb4a1f83b8f4b9dc2b71f1eaf80b into 339e92f827cd45ab8ad8d0a6a07764e0(size=12.7 K), total size for store is 24.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T22:38:57,369 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 665ebb4a1f83b8f4b9dc2b71f1eaf80b: 2024-11-10T22:38:57,369 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., storeName=665ebb4a1f83b8f4b9dc2b71f1eaf80b/A, priority=13, startTime=1731278336917; duration=0sec 2024-11-10T22:38:57,369 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T22:38:57,369 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 665ebb4a1f83b8f4b9dc2b71f1eaf80b:A 2024-11-10T22:38:57,844 INFO [Thread-1244 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1244. 2024-11-10T22:38:57,844 DEBUG [Thread-1244 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T22:38:57,844 DEBUG [Thread-1244 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:38:57,844 DEBUG [Thread-1244 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:38:57,844 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T22:38:57,854 INFO [Thread-1250 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1250. 2024-11-10T22:38:57,854 INFO [Thread-1246 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1246. 2024-11-10T22:38:57,854 INFO [Thread-1248 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1248. 2024-11-10T22:38:57,854 DEBUG [Thread-1250 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T22:38:57,854 DEBUG [Thread-1246 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T22:38:57,854 DEBUG [Thread-1250 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:38:57,854 DEBUG [Thread-1246 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:38:57,854 DEBUG [Thread-1248 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T22:38:57,854 DEBUG [Thread-1248 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:38:57,854 DEBUG [Thread-1250 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:38:57,854 DEBUG [Thread-1246 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:38:57,854 DEBUG [Thread-1248 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:38:57,854 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T22:38:57,854 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T22:38:57,854 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T22:38:57,855 INFO [Thread-1252 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1252. 2024-11-10T22:38:57,855 DEBUG [Thread-1252 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T22:38:57,855 DEBUG [Thread-1252 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:38:57,855 DEBUG [Thread-1252 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:38:57,855 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T22:38:57,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-10T22:38:57,882 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T22:38:57,882 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-10T22:38:57,883 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 54 2024-11-10T22:38:57,883 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 51 2024-11-10T22:38:57,883 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 61 2024-11-10T22:38:57,883 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 64 2024-11-10T22:38:57,883 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 50 2024-11-10T22:38:57,883 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-10T22:38:57,883 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3651 2024-11-10T22:38:57,883 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3580 2024-11-10T22:38:57,883 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3671 2024-11-10T22:38:57,883 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3661 2024-11-10T22:38:57,883 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3600 2024-11-10T22:38:57,883 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-10T22:38:57,883 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-10T22:38:57,883 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T22:38:57,883 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:38:57,883 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:38:57,883 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T22:38:57,884 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-10T22:38:57,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] procedure2.ProcedureExecutor(1139): Stored pid=92, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-10T22:38:57,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-11-10T22:38:57,886 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731278337886"}]},"ts":"1731278337886"} 2024-11-10T22:38:57,888 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-10T22:38:57,888 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-10T22:38:57,888 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=93, ppid=92, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-10T22:38:57,889 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=665ebb4a1f83b8f4b9dc2b71f1eaf80b, UNASSIGN}] 2024-11-10T22:38:57,890 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=665ebb4a1f83b8f4b9dc2b71f1eaf80b, UNASSIGN 2024-11-10T22:38:57,891 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=94 updating hbase:meta row=665ebb4a1f83b8f4b9dc2b71f1eaf80b, regionState=CLOSING, regionLocation=849d0d71310f,43753,1731278250663 2024-11-10T22:38:57,892 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=665ebb4a1f83b8f4b9dc2b71f1eaf80b, UNASSIGN because future has completed 2024-11-10T22:38:57,892 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-10T22:38:57,892 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=95, ppid=94, state=RUNNABLE, hasLock=false; CloseRegionProcedure 665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663}] 2024-11-10T22:38:57,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-11-10T22:38:58,044 INFO [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] handler.UnassignRegionHandler(122): Close 665ebb4a1f83b8f4b9dc2b71f1eaf80b 2024-11-10T22:38:58,044 DEBUG [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-10T22:38:58,044 DEBUG [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1722): Closing 665ebb4a1f83b8f4b9dc2b71f1eaf80b, disabling compactions & flushes 2024-11-10T22:38:58,044 INFO [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b. 2024-11-10T22:38:58,044 DEBUG [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b. 2024-11-10T22:38:58,044 DEBUG [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b. after waiting 0 ms 2024-11-10T22:38:58,044 DEBUG [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b. 2024-11-10T22:38:58,044 INFO [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(2902): Flushing 665ebb4a1f83b8f4b9dc2b71f1eaf80b 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-10T22:38:58,048 DEBUG [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/A/dc457b324f6847caacb3710d49d995f7 is 50, key is test_row_0/A:col10/1731278337854/Put/seqid=0 2024-11-10T22:38:58,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742168_1344 (size=9857) 2024-11-10T22:38:58,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-11-10T22:38:58,452 INFO [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=359 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/A/dc457b324f6847caacb3710d49d995f7 2024-11-10T22:38:58,458 DEBUG [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/B/7bf50b1d87de46a49f9e9b85a7b163b7 is 50, key is test_row_0/B:col10/1731278337854/Put/seqid=0 2024-11-10T22:38:58,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742169_1345 (size=9857) 2024-11-10T22:38:58,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-11-10T22:38:58,862 INFO [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=359 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/B/7bf50b1d87de46a49f9e9b85a7b163b7 2024-11-10T22:38:58,868 DEBUG [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/C/523fe288834f44649f503eff5b0ec901 is 50, key is test_row_0/C:col10/1731278337854/Put/seqid=0 2024-11-10T22:38:58,871 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742170_1346 (size=9857) 2024-11-10T22:38:59,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-11-10T22:38:59,063 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-10T22:38:59,271 INFO [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=359 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/C/523fe288834f44649f503eff5b0ec901 2024-11-10T22:38:59,275 DEBUG [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/A/dc457b324f6847caacb3710d49d995f7 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/A/dc457b324f6847caacb3710d49d995f7 2024-11-10T22:38:59,279 INFO [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/A/dc457b324f6847caacb3710d49d995f7, entries=100, sequenceid=359, filesize=9.6 K 2024-11-10T22:38:59,279 DEBUG [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/B/7bf50b1d87de46a49f9e9b85a7b163b7 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/B/7bf50b1d87de46a49f9e9b85a7b163b7 2024-11-10T22:38:59,282 INFO [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/B/7bf50b1d87de46a49f9e9b85a7b163b7, entries=100, sequenceid=359, filesize=9.6 K 2024-11-10T22:38:59,283 DEBUG [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/.tmp/C/523fe288834f44649f503eff5b0ec901 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/C/523fe288834f44649f503eff5b0ec901 2024-11-10T22:38:59,286 INFO [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/C/523fe288834f44649f503eff5b0ec901, entries=100, sequenceid=359, filesize=9.6 K 2024-11-10T22:38:59,286 INFO [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 665ebb4a1f83b8f4b9dc2b71f1eaf80b in 1242ms, sequenceid=359, compaction requested=true 2024-11-10T22:38:59,287 DEBUG [StoreCloser-TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/A/44b73e20feca4c3ca239a03cc2a1c159, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/A/94ce0a91e9654a14aed156dbdadba18a, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/A/50c70f061cee45219629bc9b670e7499, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/A/35b31f57a1ec48ec99d4e1cb8d385709, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/A/0ecc17aa9d35429b9245e68ba589fdfe, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/A/9486bfc238c741c6a3a0bf754b8fbe10, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/A/e9c98768aac84abb9b1076a9e5518e9b, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/A/63032a8b961a47c9b5fcecee5480b6a0, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/A/264503b7d4774771b53806205d18cb90, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/A/05ee1128f7d3499793978c982367885b, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/A/6b83954172ca466f8ac13274d5285c65, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/A/1a8acacf8d4f4ebbbcb91da594c0d77e, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/A/907da03b1de040bdb1bd8406609e1f42, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/A/aa73af33ebfc4e8591c8b6bf9568cac8, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/A/54dd7fea22e94151a760b8b635784d68, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/A/3c663e4a96e24a4cbb08724a4e3eaae8, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/A/915c3b12ad1f4491a2037702cf43be68, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/A/29dd34536b2f48e98771122c88dff420, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/A/c12930ee69934102b0ebdbfcfc18dd76, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/A/1b1232e2b04d4b6c9e85a6d6f10df467, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/A/2a0a9ce8ffb844f08981933b89487713, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/A/510e129d99a44fe9a46d0cd7d19bec6f] to archive 2024-11-10T22:38:59,288 DEBUG [StoreCloser-TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T22:38:59,289 DEBUG [StoreCloser-TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/A/44b73e20feca4c3ca239a03cc2a1c159 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/A/44b73e20feca4c3ca239a03cc2a1c159 2024-11-10T22:38:59,290 DEBUG [StoreCloser-TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/A/94ce0a91e9654a14aed156dbdadba18a to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/A/94ce0a91e9654a14aed156dbdadba18a 2024-11-10T22:38:59,291 DEBUG [StoreCloser-TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/A/50c70f061cee45219629bc9b670e7499 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/A/50c70f061cee45219629bc9b670e7499 2024-11-10T22:38:59,291 DEBUG [StoreCloser-TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/A/35b31f57a1ec48ec99d4e1cb8d385709 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/A/35b31f57a1ec48ec99d4e1cb8d385709 2024-11-10T22:38:59,292 DEBUG [StoreCloser-TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/A/0ecc17aa9d35429b9245e68ba589fdfe to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/A/0ecc17aa9d35429b9245e68ba589fdfe 2024-11-10T22:38:59,293 DEBUG [StoreCloser-TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/A/9486bfc238c741c6a3a0bf754b8fbe10 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/A/9486bfc238c741c6a3a0bf754b8fbe10 2024-11-10T22:38:59,294 DEBUG [StoreCloser-TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/A/e9c98768aac84abb9b1076a9e5518e9b to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/A/e9c98768aac84abb9b1076a9e5518e9b 2024-11-10T22:38:59,295 DEBUG [StoreCloser-TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/A/63032a8b961a47c9b5fcecee5480b6a0 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/A/63032a8b961a47c9b5fcecee5480b6a0 2024-11-10T22:38:59,296 DEBUG [StoreCloser-TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/A/264503b7d4774771b53806205d18cb90 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/A/264503b7d4774771b53806205d18cb90 2024-11-10T22:38:59,297 DEBUG [StoreCloser-TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/A/05ee1128f7d3499793978c982367885b to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/A/05ee1128f7d3499793978c982367885b 2024-11-10T22:38:59,298 DEBUG [StoreCloser-TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/A/6b83954172ca466f8ac13274d5285c65 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/A/6b83954172ca466f8ac13274d5285c65 2024-11-10T22:38:59,299 DEBUG [StoreCloser-TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/A/1a8acacf8d4f4ebbbcb91da594c0d77e to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/A/1a8acacf8d4f4ebbbcb91da594c0d77e 2024-11-10T22:38:59,300 DEBUG [StoreCloser-TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/A/907da03b1de040bdb1bd8406609e1f42 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/A/907da03b1de040bdb1bd8406609e1f42 2024-11-10T22:38:59,300 DEBUG [StoreCloser-TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/A/aa73af33ebfc4e8591c8b6bf9568cac8 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/A/aa73af33ebfc4e8591c8b6bf9568cac8 2024-11-10T22:38:59,301 DEBUG [StoreCloser-TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/A/54dd7fea22e94151a760b8b635784d68 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/A/54dd7fea22e94151a760b8b635784d68 2024-11-10T22:38:59,302 DEBUG [StoreCloser-TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/A/3c663e4a96e24a4cbb08724a4e3eaae8 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/A/3c663e4a96e24a4cbb08724a4e3eaae8 2024-11-10T22:38:59,303 DEBUG [StoreCloser-TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/A/915c3b12ad1f4491a2037702cf43be68 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/A/915c3b12ad1f4491a2037702cf43be68 2024-11-10T22:38:59,304 DEBUG [StoreCloser-TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/A/29dd34536b2f48e98771122c88dff420 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/A/29dd34536b2f48e98771122c88dff420 2024-11-10T22:38:59,305 DEBUG [StoreCloser-TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/A/c12930ee69934102b0ebdbfcfc18dd76 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/A/c12930ee69934102b0ebdbfcfc18dd76 2024-11-10T22:38:59,306 DEBUG [StoreCloser-TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/A/1b1232e2b04d4b6c9e85a6d6f10df467 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/A/1b1232e2b04d4b6c9e85a6d6f10df467 2024-11-10T22:38:59,306 DEBUG [StoreCloser-TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/A/2a0a9ce8ffb844f08981933b89487713 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/A/2a0a9ce8ffb844f08981933b89487713 2024-11-10T22:38:59,307 DEBUG [StoreCloser-TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/A/510e129d99a44fe9a46d0cd7d19bec6f to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/A/510e129d99a44fe9a46d0cd7d19bec6f 2024-11-10T22:38:59,309 DEBUG [StoreCloser-TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/B/807c2b356f13449ba86b4a0760000186, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/B/9271e946e0a94f538c8978b87e21ba61, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/B/76f5de29caa54e768d4ae8071d5ee438, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/B/e44438a5288143d1a4d9c6263d4c961f, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/B/babab62d94414052807fb76475c30695, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/B/414179b8c00649aea83c5f77dfce8c58, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/B/078c225608d14ac0b9582a1ba5698115, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/B/b9ef504fcdc74e62a9fc47cc64ec5e0b, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/B/a72051362a6748a4b90ced820fa6ed71, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/B/750eaf21699f4aaaa164823968eb722d, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/B/04698c3651824f2bb23f583068e74c8f, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/B/8c88bbb4edd64b3d8a998fd5c2a24663, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/B/0c741fe94efb424aa5ca9d5dfd76b3df, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/B/ddea2f2255dd4bb19eb2d1ee6dba5d09, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/B/52fc2a922a0b48858a69e505949cd5b7, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/B/5bd4b82172bf45d6bb7f557b9eead35b, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/B/f9e6f5f6884d45aba56fcb86bacb8859, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/B/f770e4eb49ff4f9a887a2151d7e81642, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/B/4e6ae51617ba4aeeadc941548d8f68bb, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/B/b5a46a25d7b74c6d873830749b481337, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/B/e1ac1973bfec48898cecda14a3e2e13a, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/B/bac4d3c649654f7fa4e73f8345fc4ea4] to archive 2024-11-10T22:38:59,309 DEBUG [StoreCloser-TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T22:38:59,310 DEBUG [StoreCloser-TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/B/807c2b356f13449ba86b4a0760000186 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/B/807c2b356f13449ba86b4a0760000186 2024-11-10T22:38:59,311 DEBUG [StoreCloser-TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/B/9271e946e0a94f538c8978b87e21ba61 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/B/9271e946e0a94f538c8978b87e21ba61 2024-11-10T22:38:59,312 DEBUG [StoreCloser-TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/B/76f5de29caa54e768d4ae8071d5ee438 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/B/76f5de29caa54e768d4ae8071d5ee438 2024-11-10T22:38:59,313 DEBUG [StoreCloser-TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/B/e44438a5288143d1a4d9c6263d4c961f to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/B/e44438a5288143d1a4d9c6263d4c961f 2024-11-10T22:38:59,314 DEBUG [StoreCloser-TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/B/babab62d94414052807fb76475c30695 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/B/babab62d94414052807fb76475c30695 2024-11-10T22:38:59,315 DEBUG [StoreCloser-TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/B/414179b8c00649aea83c5f77dfce8c58 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/B/414179b8c00649aea83c5f77dfce8c58 2024-11-10T22:38:59,316 DEBUG [StoreCloser-TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/B/078c225608d14ac0b9582a1ba5698115 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/B/078c225608d14ac0b9582a1ba5698115 2024-11-10T22:38:59,317 DEBUG [StoreCloser-TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/B/b9ef504fcdc74e62a9fc47cc64ec5e0b to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/B/b9ef504fcdc74e62a9fc47cc64ec5e0b 2024-11-10T22:38:59,318 DEBUG [StoreCloser-TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/B/a72051362a6748a4b90ced820fa6ed71 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/B/a72051362a6748a4b90ced820fa6ed71 2024-11-10T22:38:59,318 DEBUG [StoreCloser-TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/B/750eaf21699f4aaaa164823968eb722d to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/B/750eaf21699f4aaaa164823968eb722d 2024-11-10T22:38:59,319 DEBUG [StoreCloser-TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/B/04698c3651824f2bb23f583068e74c8f to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/B/04698c3651824f2bb23f583068e74c8f 2024-11-10T22:38:59,320 DEBUG [StoreCloser-TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/B/8c88bbb4edd64b3d8a998fd5c2a24663 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/B/8c88bbb4edd64b3d8a998fd5c2a24663 2024-11-10T22:38:59,321 DEBUG [StoreCloser-TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/B/0c741fe94efb424aa5ca9d5dfd76b3df to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/B/0c741fe94efb424aa5ca9d5dfd76b3df 2024-11-10T22:38:59,322 DEBUG [StoreCloser-TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/B/ddea2f2255dd4bb19eb2d1ee6dba5d09 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/B/ddea2f2255dd4bb19eb2d1ee6dba5d09 2024-11-10T22:38:59,323 DEBUG [StoreCloser-TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/B/52fc2a922a0b48858a69e505949cd5b7 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/B/52fc2a922a0b48858a69e505949cd5b7 2024-11-10T22:38:59,324 DEBUG [StoreCloser-TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/B/5bd4b82172bf45d6bb7f557b9eead35b to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/B/5bd4b82172bf45d6bb7f557b9eead35b 2024-11-10T22:38:59,325 DEBUG [StoreCloser-TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/B/f9e6f5f6884d45aba56fcb86bacb8859 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/B/f9e6f5f6884d45aba56fcb86bacb8859 2024-11-10T22:38:59,326 DEBUG [StoreCloser-TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/B/f770e4eb49ff4f9a887a2151d7e81642 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/B/f770e4eb49ff4f9a887a2151d7e81642 2024-11-10T22:38:59,326 DEBUG [StoreCloser-TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/B/4e6ae51617ba4aeeadc941548d8f68bb to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/B/4e6ae51617ba4aeeadc941548d8f68bb 2024-11-10T22:38:59,327 DEBUG [StoreCloser-TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/B/b5a46a25d7b74c6d873830749b481337 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/B/b5a46a25d7b74c6d873830749b481337 2024-11-10T22:38:59,328 DEBUG [StoreCloser-TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/B/e1ac1973bfec48898cecda14a3e2e13a to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/B/e1ac1973bfec48898cecda14a3e2e13a 2024-11-10T22:38:59,329 DEBUG [StoreCloser-TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/B/bac4d3c649654f7fa4e73f8345fc4ea4 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/B/bac4d3c649654f7fa4e73f8345fc4ea4 2024-11-10T22:38:59,331 DEBUG [StoreCloser-TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/C/7ecf0e35f2a94aafb73b2a9b2311a14c, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/C/9493540e1fb2421d9402b0c6a988fe04, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/C/63b834ae163b49619c3041236aeeb61b, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/C/987e7e90493940eca649e12e8376c494, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/C/25a2e22f46064e8dba537d1428553558, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/C/755408b8418b4b57a8bf59a80d01f098, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/C/b312b7db25014a4cbbb36a0cc71bd30d, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/C/7334e6d4fce0405a92d9edb48d45708d, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/C/40b954093197427ab7ddd7358c387eeb, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/C/b2892919e0844e1299854f4f16fe4742, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/C/18515a535c9240b394fb00a2dd2474b5, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/C/fd7b0934729341588f79e18efcd0a831, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/C/3c7ded45110944859af8bafe93df5cae, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/C/a12edef64f4a427f8638470ec07c6b45, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/C/60115c271ed145a7a381e54f0b9da706, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/C/842ee947976a414fbf62e3c9fae6cf92, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/C/a8e296c22c7e43ed8a55a72ca0d7ee40, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/C/e3a7883d66524b5ea79b1a7a16bbbd41, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/C/6f49cdc405004f96b5e6ee3a7c8d8b49, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/C/d958a5f5f88848c2a17368120136ad0e, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/C/b84d4dbf2ac34d5d8b275283f8c8a8de, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/C/21c19a5edf1f46f9bd3ae2383b7c36c4] to archive 2024-11-10T22:38:59,331 DEBUG [StoreCloser-TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T22:38:59,332 DEBUG [StoreCloser-TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/C/7ecf0e35f2a94aafb73b2a9b2311a14c to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/C/7ecf0e35f2a94aafb73b2a9b2311a14c 2024-11-10T22:38:59,333 DEBUG [StoreCloser-TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/C/9493540e1fb2421d9402b0c6a988fe04 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/C/9493540e1fb2421d9402b0c6a988fe04 2024-11-10T22:38:59,334 DEBUG [StoreCloser-TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/C/63b834ae163b49619c3041236aeeb61b to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/C/63b834ae163b49619c3041236aeeb61b 2024-11-10T22:38:59,335 DEBUG [StoreCloser-TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/C/987e7e90493940eca649e12e8376c494 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/C/987e7e90493940eca649e12e8376c494 2024-11-10T22:38:59,336 DEBUG [StoreCloser-TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/C/25a2e22f46064e8dba537d1428553558 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/C/25a2e22f46064e8dba537d1428553558 2024-11-10T22:38:59,337 DEBUG [StoreCloser-TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/C/755408b8418b4b57a8bf59a80d01f098 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/C/755408b8418b4b57a8bf59a80d01f098 2024-11-10T22:38:59,338 DEBUG [StoreCloser-TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/C/b312b7db25014a4cbbb36a0cc71bd30d to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/C/b312b7db25014a4cbbb36a0cc71bd30d 2024-11-10T22:38:59,339 DEBUG [StoreCloser-TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/C/7334e6d4fce0405a92d9edb48d45708d to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/C/7334e6d4fce0405a92d9edb48d45708d 2024-11-10T22:38:59,339 DEBUG [StoreCloser-TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/C/40b954093197427ab7ddd7358c387eeb to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/C/40b954093197427ab7ddd7358c387eeb 2024-11-10T22:38:59,340 DEBUG [StoreCloser-TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/C/b2892919e0844e1299854f4f16fe4742 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/C/b2892919e0844e1299854f4f16fe4742 2024-11-10T22:38:59,341 DEBUG [StoreCloser-TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/C/18515a535c9240b394fb00a2dd2474b5 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/C/18515a535c9240b394fb00a2dd2474b5 2024-11-10T22:38:59,342 DEBUG [StoreCloser-TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/C/fd7b0934729341588f79e18efcd0a831 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/C/fd7b0934729341588f79e18efcd0a831 2024-11-10T22:38:59,343 DEBUG [StoreCloser-TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/C/3c7ded45110944859af8bafe93df5cae to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/C/3c7ded45110944859af8bafe93df5cae 2024-11-10T22:38:59,344 DEBUG [StoreCloser-TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/C/a12edef64f4a427f8638470ec07c6b45 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/C/a12edef64f4a427f8638470ec07c6b45 2024-11-10T22:38:59,345 DEBUG [StoreCloser-TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/C/60115c271ed145a7a381e54f0b9da706 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/C/60115c271ed145a7a381e54f0b9da706 2024-11-10T22:38:59,346 DEBUG [StoreCloser-TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/C/842ee947976a414fbf62e3c9fae6cf92 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/C/842ee947976a414fbf62e3c9fae6cf92 2024-11-10T22:38:59,346 DEBUG [StoreCloser-TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/C/a8e296c22c7e43ed8a55a72ca0d7ee40 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/C/a8e296c22c7e43ed8a55a72ca0d7ee40 2024-11-10T22:38:59,347 DEBUG [StoreCloser-TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/C/e3a7883d66524b5ea79b1a7a16bbbd41 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/C/e3a7883d66524b5ea79b1a7a16bbbd41 2024-11-10T22:38:59,348 DEBUG [StoreCloser-TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/C/6f49cdc405004f96b5e6ee3a7c8d8b49 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/C/6f49cdc405004f96b5e6ee3a7c8d8b49 2024-11-10T22:38:59,349 DEBUG [StoreCloser-TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/C/d958a5f5f88848c2a17368120136ad0e to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/C/d958a5f5f88848c2a17368120136ad0e 2024-11-10T22:38:59,350 DEBUG [StoreCloser-TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/C/b84d4dbf2ac34d5d8b275283f8c8a8de to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/C/b84d4dbf2ac34d5d8b275283f8c8a8de 2024-11-10T22:38:59,351 DEBUG [StoreCloser-TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/C/21c19a5edf1f46f9bd3ae2383b7c36c4 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/C/21c19a5edf1f46f9bd3ae2383b7c36c4 2024-11-10T22:38:59,355 DEBUG [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/recovered.edits/362.seqid, newMaxSeqId=362, maxSeqId=1 2024-11-10T22:38:59,356 INFO [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b. 2024-11-10T22:38:59,356 DEBUG [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1676): Region close journal for 665ebb4a1f83b8f4b9dc2b71f1eaf80b: Waiting for close lock at 1731278338044Running coprocessor pre-close hooks at 1731278338044Disabling compacts and flushes for region at 1731278338044Disabling writes for close at 1731278338044Obtaining lock to block concurrent updates at 1731278338044Preparing flush snapshotting stores in 665ebb4a1f83b8f4b9dc2b71f1eaf80b at 1731278338044Finished memstore snapshotting TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1731278338045 (+1 ms)Flushing stores of TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b. at 1731278338045Flushing 665ebb4a1f83b8f4b9dc2b71f1eaf80b/A: creating writer at 1731278338045Flushing 665ebb4a1f83b8f4b9dc2b71f1eaf80b/A: appending metadata at 1731278338048 (+3 ms)Flushing 665ebb4a1f83b8f4b9dc2b71f1eaf80b/A: closing flushed file at 1731278338048Flushing 665ebb4a1f83b8f4b9dc2b71f1eaf80b/B: creating writer at 1731278338455 (+407 ms)Flushing 665ebb4a1f83b8f4b9dc2b71f1eaf80b/B: appending metadata at 1731278338457 (+2 ms)Flushing 665ebb4a1f83b8f4b9dc2b71f1eaf80b/B: closing flushed file at 1731278338457Flushing 665ebb4a1f83b8f4b9dc2b71f1eaf80b/C: creating writer at 1731278338865 (+408 ms)Flushing 665ebb4a1f83b8f4b9dc2b71f1eaf80b/C: appending metadata at 1731278338867 (+2 ms)Flushing 665ebb4a1f83b8f4b9dc2b71f1eaf80b/C: closing flushed file at 1731278338867Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@b857b6c: reopening flushed file at 1731278339275 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@9bac190: reopening flushed file at 1731278339279 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@77e3e0a7: reopening flushed file at 1731278339282 (+3 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 665ebb4a1f83b8f4b9dc2b71f1eaf80b in 1242ms, sequenceid=359, compaction requested=true at 1731278339286 (+4 ms)Writing region close event to WAL at 1731278339352 (+66 ms)Running coprocessor post-close hooks at 1731278339355 (+3 ms)Closed at 1731278339355 2024-11-10T22:38:59,357 INFO [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] handler.UnassignRegionHandler(157): Closed 665ebb4a1f83b8f4b9dc2b71f1eaf80b 2024-11-10T22:38:59,358 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=94 updating hbase:meta row=665ebb4a1f83b8f4b9dc2b71f1eaf80b, regionState=CLOSED 2024-11-10T22:38:59,359 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=95, ppid=94, state=RUNNABLE, hasLock=false; CloseRegionProcedure 665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 because future has completed 2024-11-10T22:38:59,361 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=95, resume processing ppid=94 2024-11-10T22:38:59,361 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, ppid=94, state=SUCCESS, hasLock=false; CloseRegionProcedure 665ebb4a1f83b8f4b9dc2b71f1eaf80b, server=849d0d71310f,43753,1731278250663 in 1.4680 sec 2024-11-10T22:38:59,363 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=93 2024-11-10T22:38:59,363 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=93, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=665ebb4a1f83b8f4b9dc2b71f1eaf80b, UNASSIGN in 1.4720 sec 2024-11-10T22:38:59,364 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=93, resume processing ppid=92 2024-11-10T22:38:59,364 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, ppid=92, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.4750 sec 2024-11-10T22:38:59,365 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731278339365"}]},"ts":"1731278339365"} 2024-11-10T22:38:59,367 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-10T22:38:59,367 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-10T22:38:59,368 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.4830 sec 2024-11-10T22:39:00,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-11-10T22:39:00,033 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-10T22:39:00,033 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-10T22:39:00,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] procedure2.ProcedureExecutor(1139): Stored pid=96, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T22:39:00,035 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=96, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T22:39:00,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=96 2024-11-10T22:39:00,036 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=96, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T22:39:00,037 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b 2024-11-10T22:39:00,039 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/A, FileablePath, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/B, FileablePath, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/C, FileablePath, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/recovered.edits] 2024-11-10T22:39:00,041 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/A/339e92f827cd45ab8ad8d0a6a07764e0 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/A/339e92f827cd45ab8ad8d0a6a07764e0 2024-11-10T22:39:00,042 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/A/66149c004a4b4fb5b41870a0dca038ba to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/A/66149c004a4b4fb5b41870a0dca038ba 2024-11-10T22:39:00,044 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/A/dc457b324f6847caacb3710d49d995f7 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/A/dc457b324f6847caacb3710d49d995f7 2024-11-10T22:39:00,045 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/B/17f15a03098045b5b9c82dc9c8a7abac to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/B/17f15a03098045b5b9c82dc9c8a7abac 2024-11-10T22:39:00,047 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/B/7bf50b1d87de46a49f9e9b85a7b163b7 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/B/7bf50b1d87de46a49f9e9b85a7b163b7 2024-11-10T22:39:00,048 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/B/c4d43dbfe382493fb071bc89944652a8 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/B/c4d43dbfe382493fb071bc89944652a8 2024-11-10T22:39:00,050 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/C/523fe288834f44649f503eff5b0ec901 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/C/523fe288834f44649f503eff5b0ec901 2024-11-10T22:39:00,051 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/C/59ddcbb53efb4cac859c1a56dbde1325 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/C/59ddcbb53efb4cac859c1a56dbde1325 2024-11-10T22:39:00,052 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/C/aa2bc932a88e42aa9c53d8ec66f05c00 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/C/aa2bc932a88e42aa9c53d8ec66f05c00 2024-11-10T22:39:00,054 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/recovered.edits/362.seqid to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b/recovered.edits/362.seqid 2024-11-10T22:39:00,055 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/665ebb4a1f83b8f4b9dc2b71f1eaf80b 2024-11-10T22:39:00,055 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-10T22:39:00,057 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=96, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T22:39:00,060 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-10T22:39:00,062 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-10T22:39:00,063 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=96, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T22:39:00,063 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-10T22:39:00,063 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731278340063"}]},"ts":"9223372036854775807"} 2024-11-10T22:39:00,065 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-10T22:39:00,065 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 665ebb4a1f83b8f4b9dc2b71f1eaf80b, NAME => 'TestAcidGuarantees,,1731278315032.665ebb4a1f83b8f4b9dc2b71f1eaf80b.', STARTKEY => '', ENDKEY => ''}] 2024-11-10T22:39:00,065 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-10T22:39:00,065 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731278340065"}]},"ts":"9223372036854775807"} 2024-11-10T22:39:00,066 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-10T22:39:00,067 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=96, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T22:39:00,068 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 33 msec 2024-11-10T22:39:00,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=96 2024-11-10T22:39:00,143 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-10T22:39:00,145 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-10T22:39:00,152 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testGetAtomicity Thread=240 (was 242), OpenFileDescriptor=451 (was 463), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=295 (was 222) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=6994 (was 7074) 2024-11-10T22:39:00,161 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobScanAtomicity Thread=240, OpenFileDescriptor=451, MaxFileDescriptor=1048576, SystemLoadAverage=295, ProcessCount=11, AvailableMemoryMB=6994 2024-11-10T22:39:00,162 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-10T22:39:00,162 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-10T22:39:00,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] procedure2.ProcedureExecutor(1139): Stored pid=97, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-10T22:39:00,164 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=97, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-10T22:39:00,164 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:00,165 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 97 2024-11-10T22:39:00,165 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=97, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-10T22:39:00,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-10T22:39:00,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742171_1347 (size=959) 2024-11-10T22:39:00,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-10T22:39:00,392 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-10T22:39:00,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-10T22:39:00,572 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 9cbf4ade36acb62156f78dbcf26aa02e, NAME => 'TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39 2024-11-10T22:39:00,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742172_1348 (size=53) 2024-11-10T22:39:00,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-10T22:39:00,977 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T22:39:00,977 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 9cbf4ade36acb62156f78dbcf26aa02e, disabling compactions & flushes 2024-11-10T22:39:00,978 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. 2024-11-10T22:39:00,978 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. 2024-11-10T22:39:00,978 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. after waiting 0 ms 2024-11-10T22:39:00,978 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. 2024-11-10T22:39:00,978 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. 2024-11-10T22:39:00,978 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 9cbf4ade36acb62156f78dbcf26aa02e: Waiting for close lock at 1731278340977Disabling compacts and flushes for region at 1731278340977Disabling writes for close at 1731278340978 (+1 ms)Writing region close event to WAL at 1731278340978Closed at 1731278340978 2024-11-10T22:39:00,979 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=97, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-10T22:39:00,979 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731278340979"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731278340979"}]},"ts":"1731278340979"} 2024-11-10T22:39:00,980 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-10T22:39:00,981 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=97, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-10T22:39:00,981 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731278340981"}]},"ts":"1731278340981"} 2024-11-10T22:39:00,982 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-10T22:39:00,982 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9cbf4ade36acb62156f78dbcf26aa02e, ASSIGN}] 2024-11-10T22:39:00,983 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=98, ppid=97, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9cbf4ade36acb62156f78dbcf26aa02e, ASSIGN 2024-11-10T22:39:00,984 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=98, ppid=97, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=9cbf4ade36acb62156f78dbcf26aa02e, ASSIGN; state=OFFLINE, location=849d0d71310f,43753,1731278250663; forceNewPlan=false, retain=false 2024-11-10T22:39:01,135 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=98 updating hbase:meta row=9cbf4ade36acb62156f78dbcf26aa02e, regionState=OPENING, regionLocation=849d0d71310f,43753,1731278250663 2024-11-10T22:39:01,136 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=98, ppid=97, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=9cbf4ade36acb62156f78dbcf26aa02e, ASSIGN because future has completed 2024-11-10T22:39:01,136 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=99, ppid=98, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663}] 2024-11-10T22:39:01,291 INFO [RS_OPEN_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. 2024-11-10T22:39:01,291 DEBUG [RS_OPEN_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegion(7752): Opening region: {ENCODED => 9cbf4ade36acb62156f78dbcf26aa02e, NAME => 'TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.', STARTKEY => '', ENDKEY => ''} 2024-11-10T22:39:01,291 DEBUG [RS_OPEN_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 9cbf4ade36acb62156f78dbcf26aa02e 2024-11-10T22:39:01,291 DEBUG [RS_OPEN_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T22:39:01,291 DEBUG [RS_OPEN_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegion(7794): checking encryption for 9cbf4ade36acb62156f78dbcf26aa02e 2024-11-10T22:39:01,291 DEBUG [RS_OPEN_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegion(7797): checking classloading for 9cbf4ade36acb62156f78dbcf26aa02e 2024-11-10T22:39:01,292 INFO [StoreOpener-9cbf4ade36acb62156f78dbcf26aa02e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 9cbf4ade36acb62156f78dbcf26aa02e 2024-11-10T22:39:01,294 INFO [StoreOpener-9cbf4ade36acb62156f78dbcf26aa02e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9cbf4ade36acb62156f78dbcf26aa02e columnFamilyName A 2024-11-10T22:39:01,294 DEBUG [StoreOpener-9cbf4ade36acb62156f78dbcf26aa02e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:01,294 INFO [StoreOpener-9cbf4ade36acb62156f78dbcf26aa02e-1 {}] regionserver.HStore(327): Store=9cbf4ade36acb62156f78dbcf26aa02e/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T22:39:01,294 INFO [StoreOpener-9cbf4ade36acb62156f78dbcf26aa02e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 9cbf4ade36acb62156f78dbcf26aa02e 2024-11-10T22:39:01,295 INFO [StoreOpener-9cbf4ade36acb62156f78dbcf26aa02e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9cbf4ade36acb62156f78dbcf26aa02e columnFamilyName B 2024-11-10T22:39:01,295 DEBUG [StoreOpener-9cbf4ade36acb62156f78dbcf26aa02e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:01,296 INFO [StoreOpener-9cbf4ade36acb62156f78dbcf26aa02e-1 {}] regionserver.HStore(327): Store=9cbf4ade36acb62156f78dbcf26aa02e/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T22:39:01,296 INFO [StoreOpener-9cbf4ade36acb62156f78dbcf26aa02e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 9cbf4ade36acb62156f78dbcf26aa02e 2024-11-10T22:39:01,296 INFO [StoreOpener-9cbf4ade36acb62156f78dbcf26aa02e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9cbf4ade36acb62156f78dbcf26aa02e columnFamilyName C 2024-11-10T22:39:01,296 DEBUG [StoreOpener-9cbf4ade36acb62156f78dbcf26aa02e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:01,297 INFO [StoreOpener-9cbf4ade36acb62156f78dbcf26aa02e-1 {}] regionserver.HStore(327): Store=9cbf4ade36acb62156f78dbcf26aa02e/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T22:39:01,297 DEBUG [RS_OPEN_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegion(1038): replaying wal for 9cbf4ade36acb62156f78dbcf26aa02e 2024-11-10T22:39:01,297 DEBUG [RS_OPEN_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e 2024-11-10T22:39:01,298 DEBUG [RS_OPEN_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e 2024-11-10T22:39:01,298 DEBUG [RS_OPEN_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegion(1048): stopping wal replay for 9cbf4ade36acb62156f78dbcf26aa02e 2024-11-10T22:39:01,298 DEBUG [RS_OPEN_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegion(1060): Cleaning up temporary data for 9cbf4ade36acb62156f78dbcf26aa02e 2024-11-10T22:39:01,299 DEBUG [RS_OPEN_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-10T22:39:01,300 DEBUG [RS_OPEN_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegion(1093): writing seq id for 9cbf4ade36acb62156f78dbcf26aa02e 2024-11-10T22:39:01,301 DEBUG [RS_OPEN_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-10T22:39:01,301 INFO [RS_OPEN_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegion(1114): Opened 9cbf4ade36acb62156f78dbcf26aa02e; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67085620, jitterRate=-3.463625907897949E-4}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-10T22:39:01,302 DEBUG [RS_OPEN_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 9cbf4ade36acb62156f78dbcf26aa02e 2024-11-10T22:39:01,302 DEBUG [RS_OPEN_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegion(1006): Region open journal for 9cbf4ade36acb62156f78dbcf26aa02e: Running coprocessor pre-open hook at 1731278341291Writing region info on filesystem at 1731278341291Initializing all the Stores at 1731278341292 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731278341292Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731278341292Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731278341292Cleaning up temporary data from old regions at 1731278341298 (+6 ms)Running coprocessor post-open hooks at 1731278341302 (+4 ms)Region opened successfully at 1731278341302 2024-11-10T22:39:01,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-10T22:39:01,303 INFO [RS_OPEN_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., pid=99, masterSystemTime=1731278341288 2024-11-10T22:39:01,304 DEBUG [RS_OPEN_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. 2024-11-10T22:39:01,304 INFO [RS_OPEN_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. 2024-11-10T22:39:01,305 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=98 updating hbase:meta row=9cbf4ade36acb62156f78dbcf26aa02e, regionState=OPEN, openSeqNum=2, regionLocation=849d0d71310f,43753,1731278250663 2024-11-10T22:39:01,306 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=99, ppid=98, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 because future has completed 2024-11-10T22:39:01,308 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=99, resume processing ppid=98 2024-11-10T22:39:01,308 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, ppid=98, state=SUCCESS, hasLock=false; OpenRegionProcedure 9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 in 171 msec 2024-11-10T22:39:01,310 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-11-10T22:39:01,310 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9cbf4ade36acb62156f78dbcf26aa02e, ASSIGN in 327 msec 2024-11-10T22:39:01,311 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=97, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-10T22:39:01,311 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731278341311"}]},"ts":"1731278341311"} 2024-11-10T22:39:01,312 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-10T22:39:01,313 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=97, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-10T22:39:01,314 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1510 sec 2024-11-10T22:39:02,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-10T22:39:02,322 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-10T22:39:02,323 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1af6f666, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:39:02,323 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849d0d71310f,43351,-1 for getting cluster id 2024-11-10T22:39:02,323 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T22:39:02,324 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59ccc382-681c-4202-a6b6-e337b1afc6c7' 2024-11-10T22:39:02,325 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T22:39:02,325 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59ccc382-681c-4202-a6b6-e337b1afc6c7" 2024-11-10T22:39:02,325 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@444f91a5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:39:02,325 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849d0d71310f,43351,-1] 2024-11-10T22:39:02,325 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T22:39:02,325 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:39:02,326 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40052, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T22:39:02,327 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ae42535, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:39:02,327 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T22:39:02,328 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849d0d71310f,43753,1731278250663, seqNum=-1] 2024-11-10T22:39:02,328 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T22:39:02,328 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60782, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T22:39:02,329 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-10T22:39:02,330 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 849d0d71310f,43351,1731278249979 2024-11-10T22:39:02,330 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@124978f7 2024-11-10T22:39:02,330 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-10T22:39:02,330 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40056, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-10T22:39:02,332 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-10T22:39:02,332 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-10T22:39:02,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] procedure2.ProcedureExecutor(1139): Stored pid=100, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-10T22:39:02,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742173_1349 (size=995) 2024-11-10T22:39:02,743 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-10T22:39:02,743 INFO [PEWorker-4 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-10T22:39:02,745 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=101, ppid=100, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-10T22:39:02,747 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9cbf4ade36acb62156f78dbcf26aa02e, REOPEN/MOVE}] 2024-11-10T22:39:02,748 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=102, ppid=101, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9cbf4ade36acb62156f78dbcf26aa02e, REOPEN/MOVE 2024-11-10T22:39:02,749 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=102 updating hbase:meta row=9cbf4ade36acb62156f78dbcf26aa02e, regionState=CLOSING, regionLocation=849d0d71310f,43753,1731278250663 2024-11-10T22:39:02,750 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=102, ppid=101, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=9cbf4ade36acb62156f78dbcf26aa02e, REOPEN/MOVE because future has completed 2024-11-10T22:39:02,750 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-10T22:39:02,750 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=103, ppid=102, state=RUNNABLE, hasLock=false; CloseRegionProcedure 9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663}] 2024-11-10T22:39:02,902 INFO [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=103}] handler.UnassignRegionHandler(122): Close 9cbf4ade36acb62156f78dbcf26aa02e 2024-11-10T22:39:02,902 DEBUG [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=103}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-10T22:39:02,902 DEBUG [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=103}] regionserver.HRegion(1722): Closing 9cbf4ade36acb62156f78dbcf26aa02e, disabling compactions & flushes 2024-11-10T22:39:02,902 INFO [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=103}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. 2024-11-10T22:39:02,902 DEBUG [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=103}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. 2024-11-10T22:39:02,902 DEBUG [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=103}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. after waiting 0 ms 2024-11-10T22:39:02,902 DEBUG [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=103}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. 2024-11-10T22:39:02,905 DEBUG [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=103}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-10T22:39:02,906 INFO [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=103}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. 2024-11-10T22:39:02,906 DEBUG [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=103}] regionserver.HRegion(1676): Region close journal for 9cbf4ade36acb62156f78dbcf26aa02e: Waiting for close lock at 1731278342902Running coprocessor pre-close hooks at 1731278342902Disabling compacts and flushes for region at 1731278342902Disabling writes for close at 1731278342902Writing region close event to WAL at 1731278342903 (+1 ms)Running coprocessor post-close hooks at 1731278342906 (+3 ms)Closed at 1731278342906 2024-11-10T22:39:02,906 WARN [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=103}] regionserver.HRegionServer(3299): Not adding moved region record: 9cbf4ade36acb62156f78dbcf26aa02e to self. 2024-11-10T22:39:02,908 INFO [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=103}] handler.UnassignRegionHandler(157): Closed 9cbf4ade36acb62156f78dbcf26aa02e 2024-11-10T22:39:02,908 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=102 updating hbase:meta row=9cbf4ade36acb62156f78dbcf26aa02e, regionState=CLOSED 2024-11-10T22:39:02,909 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=103, ppid=102, state=RUNNABLE, hasLock=false; CloseRegionProcedure 9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 because future has completed 2024-11-10T22:39:02,911 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=103, resume processing ppid=102 2024-11-10T22:39:02,911 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, ppid=102, state=SUCCESS, hasLock=false; CloseRegionProcedure 9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 in 160 msec 2024-11-10T22:39:02,912 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=102, ppid=101, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=9cbf4ade36acb62156f78dbcf26aa02e, REOPEN/MOVE; state=CLOSED, location=849d0d71310f,43753,1731278250663; forceNewPlan=false, retain=true 2024-11-10T22:39:03,062 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=102 updating hbase:meta row=9cbf4ade36acb62156f78dbcf26aa02e, regionState=OPENING, regionLocation=849d0d71310f,43753,1731278250663 2024-11-10T22:39:03,064 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=102, ppid=101, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=9cbf4ade36acb62156f78dbcf26aa02e, REOPEN/MOVE because future has completed 2024-11-10T22:39:03,064 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=102, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663}] 2024-11-10T22:39:03,218 INFO [RS_OPEN_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. 2024-11-10T22:39:03,218 DEBUG [RS_OPEN_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegion(7752): Opening region: {ENCODED => 9cbf4ade36acb62156f78dbcf26aa02e, NAME => 'TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.', STARTKEY => '', ENDKEY => ''} 2024-11-10T22:39:03,219 DEBUG [RS_OPEN_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 9cbf4ade36acb62156f78dbcf26aa02e 2024-11-10T22:39:03,219 DEBUG [RS_OPEN_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T22:39:03,219 DEBUG [RS_OPEN_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegion(7794): checking encryption for 9cbf4ade36acb62156f78dbcf26aa02e 2024-11-10T22:39:03,219 DEBUG [RS_OPEN_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegion(7797): checking classloading for 9cbf4ade36acb62156f78dbcf26aa02e 2024-11-10T22:39:03,220 INFO [StoreOpener-9cbf4ade36acb62156f78dbcf26aa02e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 9cbf4ade36acb62156f78dbcf26aa02e 2024-11-10T22:39:03,221 INFO [StoreOpener-9cbf4ade36acb62156f78dbcf26aa02e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9cbf4ade36acb62156f78dbcf26aa02e columnFamilyName A 2024-11-10T22:39:03,221 DEBUG [StoreOpener-9cbf4ade36acb62156f78dbcf26aa02e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:03,222 INFO [StoreOpener-9cbf4ade36acb62156f78dbcf26aa02e-1 {}] regionserver.HStore(327): Store=9cbf4ade36acb62156f78dbcf26aa02e/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T22:39:03,222 INFO [StoreOpener-9cbf4ade36acb62156f78dbcf26aa02e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 9cbf4ade36acb62156f78dbcf26aa02e 2024-11-10T22:39:03,223 INFO [StoreOpener-9cbf4ade36acb62156f78dbcf26aa02e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9cbf4ade36acb62156f78dbcf26aa02e columnFamilyName B 2024-11-10T22:39:03,223 DEBUG [StoreOpener-9cbf4ade36acb62156f78dbcf26aa02e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:03,223 INFO [StoreOpener-9cbf4ade36acb62156f78dbcf26aa02e-1 {}] regionserver.HStore(327): Store=9cbf4ade36acb62156f78dbcf26aa02e/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T22:39:03,223 INFO [StoreOpener-9cbf4ade36acb62156f78dbcf26aa02e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 9cbf4ade36acb62156f78dbcf26aa02e 2024-11-10T22:39:03,224 INFO [StoreOpener-9cbf4ade36acb62156f78dbcf26aa02e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9cbf4ade36acb62156f78dbcf26aa02e columnFamilyName C 2024-11-10T22:39:03,224 DEBUG [StoreOpener-9cbf4ade36acb62156f78dbcf26aa02e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:03,224 INFO [StoreOpener-9cbf4ade36acb62156f78dbcf26aa02e-1 {}] regionserver.HStore(327): Store=9cbf4ade36acb62156f78dbcf26aa02e/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T22:39:03,225 DEBUG [RS_OPEN_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegion(1038): replaying wal for 9cbf4ade36acb62156f78dbcf26aa02e 2024-11-10T22:39:03,225 DEBUG [RS_OPEN_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e 2024-11-10T22:39:03,226 DEBUG [RS_OPEN_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e 2024-11-10T22:39:03,226 DEBUG [RS_OPEN_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegion(1048): stopping wal replay for 9cbf4ade36acb62156f78dbcf26aa02e 2024-11-10T22:39:03,226 DEBUG [RS_OPEN_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegion(1060): Cleaning up temporary data for 9cbf4ade36acb62156f78dbcf26aa02e 2024-11-10T22:39:03,227 DEBUG [RS_OPEN_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-10T22:39:03,228 DEBUG [RS_OPEN_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegion(1093): writing seq id for 9cbf4ade36acb62156f78dbcf26aa02e 2024-11-10T22:39:03,228 INFO [RS_OPEN_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegion(1114): Opened 9cbf4ade36acb62156f78dbcf26aa02e; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61844342, jitterRate=-0.07844749093055725}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-10T22:39:03,228 DEBUG [RS_OPEN_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 9cbf4ade36acb62156f78dbcf26aa02e 2024-11-10T22:39:03,229 DEBUG [RS_OPEN_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegion(1006): Region open journal for 9cbf4ade36acb62156f78dbcf26aa02e: Running coprocessor pre-open hook at 1731278343219Writing region info on filesystem at 1731278343219Initializing all the Stores at 1731278343219Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731278343219Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731278343220 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731278343220Cleaning up temporary data from old regions at 1731278343226 (+6 ms)Running coprocessor post-open hooks at 1731278343228 (+2 ms)Region opened successfully at 1731278343229 (+1 ms) 2024-11-10T22:39:03,230 INFO [RS_OPEN_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., pid=104, masterSystemTime=1731278343216 2024-11-10T22:39:03,231 DEBUG [RS_OPEN_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. 2024-11-10T22:39:03,231 INFO [RS_OPEN_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. 2024-11-10T22:39:03,232 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=102 updating hbase:meta row=9cbf4ade36acb62156f78dbcf26aa02e, regionState=OPEN, openSeqNum=5, regionLocation=849d0d71310f,43753,1731278250663 2024-11-10T22:39:03,233 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=104, ppid=102, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 because future has completed 2024-11-10T22:39:03,235 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=102 2024-11-10T22:39:03,235 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=102, state=SUCCESS, hasLock=false; OpenRegionProcedure 9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 in 169 msec 2024-11-10T22:39:03,236 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-11-10T22:39:03,236 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9cbf4ade36acb62156f78dbcf26aa02e, REOPEN/MOVE in 488 msec 2024-11-10T22:39:03,238 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=101, resume processing ppid=100 2024-11-10T22:39:03,238 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, ppid=100, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 492 msec 2024-11-10T22:39:03,240 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 907 msec 2024-11-10T22:39:03,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=100 2024-11-10T22:39:03,241 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-10T22:39:03,241 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9ff38b5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:39:03,242 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849d0d71310f,43351,-1 for getting cluster id 2024-11-10T22:39:03,242 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T22:39:03,242 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59ccc382-681c-4202-a6b6-e337b1afc6c7' 2024-11-10T22:39:03,243 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T22:39:03,243 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59ccc382-681c-4202-a6b6-e337b1afc6c7" 2024-11-10T22:39:03,243 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@422e3895, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:39:03,243 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849d0d71310f,43351,-1] 2024-11-10T22:39:03,243 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T22:39:03,243 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:39:03,244 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40072, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T22:39:03,244 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@48d46bdf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:39:03,245 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62ec6412, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:39:03,245 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849d0d71310f,43351,-1 for getting cluster id 2024-11-10T22:39:03,245 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T22:39:03,246 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59ccc382-681c-4202-a6b6-e337b1afc6c7' 2024-11-10T22:39:03,246 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T22:39:03,246 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59ccc382-681c-4202-a6b6-e337b1afc6c7" 2024-11-10T22:39:03,246 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66e91d88, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:39:03,246 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849d0d71310f,43351,-1] 2024-11-10T22:39:03,246 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T22:39:03,246 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:39:03,247 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40098, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T22:39:03,247 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@461066a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:39:03,248 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37f2378c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:39:03,248 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849d0d71310f,43351,-1 for getting cluster id 2024-11-10T22:39:03,248 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T22:39:03,248 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59ccc382-681c-4202-a6b6-e337b1afc6c7' 2024-11-10T22:39:03,249 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T22:39:03,249 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59ccc382-681c-4202-a6b6-e337b1afc6c7" 2024-11-10T22:39:03,249 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@309f3589, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:39:03,249 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849d0d71310f,43351,-1] 2024-11-10T22:39:03,249 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T22:39:03,249 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:39:03,249 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40108, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T22:39:03,250 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@439127ac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:39:03,250 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43cc49f7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:39:03,250 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849d0d71310f,43351,-1 for getting cluster id 2024-11-10T22:39:03,251 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T22:39:03,251 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59ccc382-681c-4202-a6b6-e337b1afc6c7' 2024-11-10T22:39:03,251 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T22:39:03,251 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59ccc382-681c-4202-a6b6-e337b1afc6c7" 2024-11-10T22:39:03,251 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@641edbb8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:39:03,251 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849d0d71310f,43351,-1] 2024-11-10T22:39:03,252 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T22:39:03,252 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:39:03,252 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40114, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T22:39:03,253 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2cb40684, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:39:03,253 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e0f0eef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:39:03,253 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849d0d71310f,43351,-1 for getting cluster id 2024-11-10T22:39:03,253 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T22:39:03,254 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59ccc382-681c-4202-a6b6-e337b1afc6c7' 2024-11-10T22:39:03,254 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T22:39:03,254 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59ccc382-681c-4202-a6b6-e337b1afc6c7" 2024-11-10T22:39:03,254 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a4ea0d4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:39:03,254 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849d0d71310f,43351,-1] 2024-11-10T22:39:03,254 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T22:39:03,254 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:39:03,255 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40134, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T22:39:03,255 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29d73cd8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:39:03,256 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65616af9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:39:03,256 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849d0d71310f,43351,-1 for getting cluster id 2024-11-10T22:39:03,256 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T22:39:03,256 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59ccc382-681c-4202-a6b6-e337b1afc6c7' 2024-11-10T22:39:03,256 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T22:39:03,257 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59ccc382-681c-4202-a6b6-e337b1afc6c7" 2024-11-10T22:39:03,257 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3996c7c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:39:03,257 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849d0d71310f,43351,-1] 2024-11-10T22:39:03,257 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T22:39:03,257 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:39:03,257 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40150, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T22:39:03,258 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d0a246e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:39:03,258 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74481d40, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:39:03,258 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849d0d71310f,43351,-1 for getting cluster id 2024-11-10T22:39:03,258 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T22:39:03,259 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59ccc382-681c-4202-a6b6-e337b1afc6c7' 2024-11-10T22:39:03,259 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T22:39:03,259 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59ccc382-681c-4202-a6b6-e337b1afc6c7" 2024-11-10T22:39:03,259 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@785bd7a0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:39:03,259 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849d0d71310f,43351,-1] 2024-11-10T22:39:03,259 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T22:39:03,260 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:39:03,260 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40176, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T22:39:03,261 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ec33f20, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:39:03,261 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@514b3d51, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:39:03,261 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849d0d71310f,43351,-1 for getting cluster id 2024-11-10T22:39:03,261 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T22:39:03,262 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59ccc382-681c-4202-a6b6-e337b1afc6c7' 2024-11-10T22:39:03,262 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T22:39:03,262 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59ccc382-681c-4202-a6b6-e337b1afc6c7" 2024-11-10T22:39:03,262 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a644c84, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:39:03,262 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849d0d71310f,43351,-1] 2024-11-10T22:39:03,262 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T22:39:03,262 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:39:03,263 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40200, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T22:39:03,263 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f247b86, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:39:03,264 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b14e3fc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:39:03,264 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849d0d71310f,43351,-1 for getting cluster id 2024-11-10T22:39:03,264 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T22:39:03,264 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59ccc382-681c-4202-a6b6-e337b1afc6c7' 2024-11-10T22:39:03,264 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T22:39:03,264 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59ccc382-681c-4202-a6b6-e337b1afc6c7" 2024-11-10T22:39:03,265 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6828122c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:39:03,265 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849d0d71310f,43351,-1] 2024-11-10T22:39:03,265 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T22:39:03,265 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:39:03,265 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40214, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T22:39:03,266 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14152c10, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:39:03,266 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75e264c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:39:03,266 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849d0d71310f,43351,-1 for getting cluster id 2024-11-10T22:39:03,266 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T22:39:03,267 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59ccc382-681c-4202-a6b6-e337b1afc6c7' 2024-11-10T22:39:03,267 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T22:39:03,267 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59ccc382-681c-4202-a6b6-e337b1afc6c7" 2024-11-10T22:39:03,267 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e4ac854, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:39:03,267 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849d0d71310f,43351,-1] 2024-11-10T22:39:03,267 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T22:39:03,267 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:39:03,268 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40234, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T22:39:03,269 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56e79b6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:39:03,269 DEBUG [Thread-1584 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T22:39:03,269 DEBUG [Thread-1603 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T22:39:03,269 DEBUG [Thread-1588 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T22:39:03,269 DEBUG [Thread-1597 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T22:39:03,269 DEBUG [Thread-1601 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T22:39:03,270 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849d0d71310f,43753,1731278250663, seqNum=-1] 2024-11-10T22:39:03,270 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849d0d71310f,43753,1731278250663, seqNum=-1] 2024-11-10T22:39:03,270 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849d0d71310f,43753,1731278250663, seqNum=-1] 2024-11-10T22:39:03,270 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849d0d71310f,43753,1731278250663, seqNum=-1] 2024-11-10T22:39:03,270 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T22:39:03,270 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T22:39:03,270 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T22:39:03,270 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T22:39:03,270 DEBUG [Thread-1592 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T22:39:03,270 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849d0d71310f,43753,1731278250663, seqNum=-1] 2024-11-10T22:39:03,271 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43351 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T22:39:03,271 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T22:39:03,271 DEBUG [Thread-1595 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T22:39:03,271 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849d0d71310f,43753,1731278250663, seqNum=-1] 2024-11-10T22:39:03,271 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60796, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T22:39:03,271 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60820, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T22:39:03,271 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T22:39:03,272 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60834, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T22:39:03,272 DEBUG [Thread-1599 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T22:39:03,272 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849d0d71310f,43753,1731278250663, seqNum=-1] 2024-11-10T22:39:03,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43351 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees 2024-11-10T22:39:03,272 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T22:39:03,272 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60844, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T22:39:03,272 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60810, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T22:39:03,273 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60840, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T22:39:03,273 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849d0d71310f,43753,1731278250663, seqNum=-1] 2024-11-10T22:39:03,273 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T22:39:03,273 DEBUG [Thread-1586 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T22:39:03,273 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60850, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T22:39:03,273 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T22:39:03,274 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60854, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T22:39:03,274 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849d0d71310f,43753,1731278250663, seqNum=-1] 2024-11-10T22:39:03,274 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T22:39:03,274 DEBUG [Thread-1590 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T22:39:03,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-10T22:39:03,275 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T22:39:03,275 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T22:39:03,275 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849d0d71310f,43753,1731278250663, seqNum=-1] 2024-11-10T22:39:03,275 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60864, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T22:39:03,275 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T22:39:03,276 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60874, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T22:39:03,276 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5] 2024-11-10T22:39:03,277 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5] 2024-11-10T22:39:03,278 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5] 2024-11-10T22:39:03,279 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5] 2024-11-10T22:39:03,279 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5] 2024-11-10T22:39:03,279 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5] 2024-11-10T22:39:03,280 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5] 2024-11-10T22:39:03,280 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5] 2024-11-10T22:39:03,282 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5] 2024-11-10T22:39:03,284 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5] 2024-11-10T22:39:03,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(8855): Flush requested on 9cbf4ade36acb62156f78dbcf26aa02e 2024-11-10T22:39:03,288 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T22:39:03,288 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9cbf4ade36acb62156f78dbcf26aa02e 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-10T22:39:03,316 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411107fef7a87e32041ccb2c53748815339b0_9cbf4ade36acb62156f78dbcf26aa02e is 50, key is test_row_0/A:col10/1731278343285/Put/seqid=0 2024-11-10T22:39:03,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742174_1350 (size=14594) 2024-11-10T22:39:03,336 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:03,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60864 deadline: 1731278403326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:03,337 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:03,337 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:03,337 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:03,337 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:03,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60844 deadline: 1731278403329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:03,337 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:03,337 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:03,338 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:03,339 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:03,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60874 deadline: 1731278403331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:03,339 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:03,339 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:03,339 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:03,344 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:03,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60834 deadline: 1731278403332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:03,344 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:03,344 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:03,344 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:03,344 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:03,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60796 deadline: 1731278403334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:03,345 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:03,345 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:03,345 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:03,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-10T22:39:03,429 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-10T22:39:03,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. 2024-11-10T22:39:03,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. as already flushing 2024-11-10T22:39:03,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. 2024-11-10T22:39:03,429 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 9cbf4ade36acb62156f78dbcf26aa02e, NAME => 'TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:03,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 9cbf4ade36acb62156f78dbcf26aa02e, NAME => 'TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:03,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9cbf4ade36acb62156f78dbcf26aa02e, NAME => 'TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9cbf4ade36acb62156f78dbcf26aa02e, NAME => 'TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:03,448 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:03,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60864 deadline: 1731278403442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:03,448 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:03,448 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:03,448 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:03,448 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:03,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60844 deadline: 1731278403442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:03,449 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:03,449 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:03,449 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:03,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60874 deadline: 1731278403442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:03,449 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:03,449 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:03,449 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:03,449 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:03,459 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:03,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60834 deadline: 1731278403452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:03,460 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:03,460 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:03,460 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:03,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60796 deadline: 1731278403452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:03,460 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:03,460 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:03,460 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:03,460 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:03,582 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-10T22:39:03,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. 2024-11-10T22:39:03,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. as already flushing 2024-11-10T22:39:03,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. 2024-11-10T22:39:03,583 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 9cbf4ade36acb62156f78dbcf26aa02e, NAME => 'TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:03,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 9cbf4ade36acb62156f78dbcf26aa02e, NAME => 'TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:03,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9cbf4ade36acb62156f78dbcf26aa02e, NAME => 'TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9cbf4ade36acb62156f78dbcf26aa02e, NAME => 'TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:03,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-10T22:39:03,659 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:03,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60864 deadline: 1731278403652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:03,659 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:03,660 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:03,660 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:03,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60844 deadline: 1731278403652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:03,660 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:03,660 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:03,660 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:03,660 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:03,660 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:03,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60874 deadline: 1731278403652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:03,660 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:03,660 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:03,660 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:03,671 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:03,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60834 deadline: 1731278403663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:03,672 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:03,672 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:03,672 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:03,672 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:03,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60796 deadline: 1731278403663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:03,672 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:03,672 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:03,672 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:03,725 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:03,729 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411107fef7a87e32041ccb2c53748815339b0_9cbf4ade36acb62156f78dbcf26aa02e to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411107fef7a87e32041ccb2c53748815339b0_9cbf4ade36acb62156f78dbcf26aa02e 2024-11-10T22:39:03,730 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/A/ebc3917ea86d4b548ba7ba1f496355cc, store: [table=TestAcidGuarantees family=A region=9cbf4ade36acb62156f78dbcf26aa02e] 2024-11-10T22:39:03,730 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/A/ebc3917ea86d4b548ba7ba1f496355cc is 175, key is test_row_0/A:col10/1731278343285/Put/seqid=0 2024-11-10T22:39:03,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742175_1351 (size=39549) 2024-11-10T22:39:03,736 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-10T22:39:03,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. 2024-11-10T22:39:03,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. as already flushing 2024-11-10T22:39:03,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. 2024-11-10T22:39:03,736 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 9cbf4ade36acb62156f78dbcf26aa02e, NAME => 'TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:03,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 9cbf4ade36acb62156f78dbcf26aa02e, NAME => 'TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:03,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9cbf4ade36acb62156f78dbcf26aa02e, NAME => 'TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9cbf4ade36acb62156f78dbcf26aa02e, NAME => 'TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:03,889 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-10T22:39:03,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. 2024-11-10T22:39:03,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. as already flushing 2024-11-10T22:39:03,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. 2024-11-10T22:39:03,889 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 9cbf4ade36acb62156f78dbcf26aa02e, NAME => 'TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:03,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 9cbf4ade36acb62156f78dbcf26aa02e, NAME => 'TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:03,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9cbf4ade36acb62156f78dbcf26aa02e, NAME => 'TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9cbf4ade36acb62156f78dbcf26aa02e, NAME => 'TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:03,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-10T22:39:03,979 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:03,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60864 deadline: 1731278403972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:03,979 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:03,980 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:03,980 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:03,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60844 deadline: 1731278403972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:03,980 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:03,980 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:03,980 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:03,980 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:03,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60874 deadline: 1731278403972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:03,980 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:03,980 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:03,980 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:03,980 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:03,990 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:03,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60834 deadline: 1731278403983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:03,991 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:03,991 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:03,991 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:03,991 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:03,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60796 deadline: 1731278403983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:03,992 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:03,992 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:03,992 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:04,042 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-10T22:39:04,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. 2024-11-10T22:39:04,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. as already flushing 2024-11-10T22:39:04,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. 2024-11-10T22:39:04,042 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 9cbf4ade36acb62156f78dbcf26aa02e, NAME => 'TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:04,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 9cbf4ade36acb62156f78dbcf26aa02e, NAME => 'TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:04,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9cbf4ade36acb62156f78dbcf26aa02e, NAME => 'TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9cbf4ade36acb62156f78dbcf26aa02e, NAME => 'TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:04,134 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/A/ebc3917ea86d4b548ba7ba1f496355cc 2024-11-10T22:39:04,159 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/B/df23d130d3de4aca8cd73a2c83ca92ad is 50, key is test_row_0/B:col10/1731278343285/Put/seqid=0 2024-11-10T22:39:04,163 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742176_1352 (size=12001) 2024-11-10T22:39:04,195 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-10T22:39:04,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. 2024-11-10T22:39:04,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. as already flushing 2024-11-10T22:39:04,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. 2024-11-10T22:39:04,196 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 9cbf4ade36acb62156f78dbcf26aa02e, NAME => 'TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:04,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 9cbf4ade36acb62156f78dbcf26aa02e, NAME => 'TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:04,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9cbf4ade36acb62156f78dbcf26aa02e, NAME => 'TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9cbf4ade36acb62156f78dbcf26aa02e, NAME => 'TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:04,348 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-10T22:39:04,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. 2024-11-10T22:39:04,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. as already flushing 2024-11-10T22:39:04,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. 2024-11-10T22:39:04,349 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 9cbf4ade36acb62156f78dbcf26aa02e, NAME => 'TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:04,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 9cbf4ade36acb62156f78dbcf26aa02e, NAME => 'TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:04,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9cbf4ade36acb62156f78dbcf26aa02e, NAME => 'TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9cbf4ade36acb62156f78dbcf26aa02e, NAME => 'TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:04,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-10T22:39:04,490 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:04,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60864 deadline: 1731278404483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:04,490 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:04,490 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:04,490 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:04,491 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:04,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60844 deadline: 1731278404483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:04,491 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:04,492 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:04,492 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:04,500 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:04,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60874 deadline: 1731278404492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:04,501 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:04,501 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:04,501 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:04,501 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-10T22:39:04,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. 2024-11-10T22:39:04,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. as already flushing 2024-11-10T22:39:04,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. 2024-11-10T22:39:04,502 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 9cbf4ade36acb62156f78dbcf26aa02e, NAME => 'TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:04,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 9cbf4ade36acb62156f78dbcf26aa02e, NAME => 'TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:04,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9cbf4ade36acb62156f78dbcf26aa02e, NAME => 'TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9cbf4ade36acb62156f78dbcf26aa02e, NAME => 'TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:04,504 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:04,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60834 deadline: 1731278404502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:04,505 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:04,505 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:04,505 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:04,505 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:04,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60796 deadline: 1731278404503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:04,506 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:04,506 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:04,506 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:04,563 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/B/df23d130d3de4aca8cd73a2c83ca92ad 2024-11-10T22:39:04,592 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/C/0d5ae8554bed48978fd73b35e356403c is 50, key is test_row_0/C:col10/1731278343285/Put/seqid=0 2024-11-10T22:39:04,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742177_1353 (size=12001) 2024-11-10T22:39:04,655 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-10T22:39:04,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. 2024-11-10T22:39:04,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. as already flushing 2024-11-10T22:39:04,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. 2024-11-10T22:39:04,655 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 9cbf4ade36acb62156f78dbcf26aa02e, NAME => 'TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:04,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 9cbf4ade36acb62156f78dbcf26aa02e, NAME => 'TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:04,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9cbf4ade36acb62156f78dbcf26aa02e, NAME => 'TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9cbf4ade36acb62156f78dbcf26aa02e, NAME => 'TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:04,808 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-10T22:39:04,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. 2024-11-10T22:39:04,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. as already flushing 2024-11-10T22:39:04,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. 2024-11-10T22:39:04,809 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 9cbf4ade36acb62156f78dbcf26aa02e, NAME => 'TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:04,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 9cbf4ade36acb62156f78dbcf26aa02e, NAME => 'TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:04,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9cbf4ade36acb62156f78dbcf26aa02e, NAME => 'TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9cbf4ade36acb62156f78dbcf26aa02e, NAME => 'TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:04,886 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-10T22:39:04,962 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-10T22:39:04,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. 2024-11-10T22:39:04,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. as already flushing 2024-11-10T22:39:04,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. 2024-11-10T22:39:04,962 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 9cbf4ade36acb62156f78dbcf26aa02e, NAME => 'TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:04,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 9cbf4ade36acb62156f78dbcf26aa02e, NAME => 'TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:04,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9cbf4ade36acb62156f78dbcf26aa02e, NAME => 'TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9cbf4ade36acb62156f78dbcf26aa02e, NAME => 'TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:04,996 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/C/0d5ae8554bed48978fd73b35e356403c 2024-11-10T22:39:05,001 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/A/ebc3917ea86d4b548ba7ba1f496355cc as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/A/ebc3917ea86d4b548ba7ba1f496355cc 2024-11-10T22:39:05,004 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/A/ebc3917ea86d4b548ba7ba1f496355cc, entries=200, sequenceid=16, filesize=38.6 K 2024-11-10T22:39:05,005 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/B/df23d130d3de4aca8cd73a2c83ca92ad as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/B/df23d130d3de4aca8cd73a2c83ca92ad 2024-11-10T22:39:05,008 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/B/df23d130d3de4aca8cd73a2c83ca92ad, entries=150, sequenceid=16, filesize=11.7 K 2024-11-10T22:39:05,009 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/C/0d5ae8554bed48978fd73b35e356403c as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/C/0d5ae8554bed48978fd73b35e356403c 2024-11-10T22:39:05,013 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/C/0d5ae8554bed48978fd73b35e356403c, entries=150, sequenceid=16, filesize=11.7 K 2024-11-10T22:39:05,014 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 9cbf4ade36acb62156f78dbcf26aa02e in 1726ms, sequenceid=16, compaction requested=false 2024-11-10T22:39:05,014 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9cbf4ade36acb62156f78dbcf26aa02e: 2024-11-10T22:39:05,115 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-10T22:39:05,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. 2024-11-10T22:39:05,115 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2902): Flushing 9cbf4ade36acb62156f78dbcf26aa02e 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-10T22:39:05,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110570b908fcb42426299268f00bc40ff4d_9cbf4ade36acb62156f78dbcf26aa02e is 50, key is test_row_0/A:col10/1731278343332/Put/seqid=0 2024-11-10T22:39:05,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742178_1354 (size=12154) 2024-11-10T22:39:05,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-10T22:39:05,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(8855): Flush requested on 9cbf4ade36acb62156f78dbcf26aa02e 2024-11-10T22:39:05,508 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. as already flushing 2024-11-10T22:39:05,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:05,532 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110570b908fcb42426299268f00bc40ff4d_9cbf4ade36acb62156f78dbcf26aa02e to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110570b908fcb42426299268f00bc40ff4d_9cbf4ade36acb62156f78dbcf26aa02e 2024-11-10T22:39:05,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/A/36bc770484ee4d2983b226fcb614b6f4, store: [table=TestAcidGuarantees family=A region=9cbf4ade36acb62156f78dbcf26aa02e] 2024-11-10T22:39:05,533 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:05,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/A/36bc770484ee4d2983b226fcb614b6f4 is 175, key is test_row_0/A:col10/1731278343332/Put/seqid=0 2024-11-10T22:39:05,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60874 deadline: 1731278405518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:05,534 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:05,534 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:05,534 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:05,534 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:05,534 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:05,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60834 deadline: 1731278405519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:05,534 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:05,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60844 deadline: 1731278405521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:05,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60796 deadline: 1731278405520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:05,534 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:05,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60864 deadline: 1731278405523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:05,534 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:05,534 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:05,534 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:05,534 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:05,534 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:05,534 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:05,535 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:05,535 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:05,535 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:05,535 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:05,535 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:05,535 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:05,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742179_1355 (size=30955) 2024-11-10T22:39:05,650 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:05,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60874 deadline: 1731278405642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:05,651 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:05,651 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:05,651 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:05,651 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:05,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60834 deadline: 1731278405642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:05,651 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:05,651 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:05,651 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:05,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60844 deadline: 1731278405642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:05,651 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:05,652 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:05,652 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:05,652 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:05,652 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:05,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60864 deadline: 1731278405642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:05,652 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:05,652 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:05,652 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:05,652 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:05,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60796 deadline: 1731278405642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:05,652 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:05,652 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:05,652 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:05,872 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:05,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60874 deadline: 1731278405862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:05,872 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:05,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60834 deadline: 1731278405862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:05,872 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:05,872 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:05,872 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:05,872 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:05,872 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:05,872 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:05,873 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:05,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60844 deadline: 1731278405862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:05,873 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:05,873 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:05,873 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:05,873 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:05,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60864 deadline: 1731278405862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:05,873 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:05,873 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:05,873 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:05,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60796 deadline: 1731278405862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:05,873 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:05,873 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:05,874 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:05,874 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:05,939 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=40, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/A/36bc770484ee4d2983b226fcb614b6f4 2024-11-10T22:39:05,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/B/584f11543f0449d281f60660321bd01f is 50, key is test_row_0/B:col10/1731278343332/Put/seqid=0 2024-11-10T22:39:05,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742180_1356 (size=12001) 2024-11-10T22:39:06,192 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:06,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60874 deadline: 1731278406183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:06,193 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:06,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60834 deadline: 1731278406183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:06,193 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:06,193 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:06,193 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:06,193 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:06,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60844 deadline: 1731278406183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:06,193 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:06,193 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:06,193 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:06,193 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:06,193 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:06,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60864 deadline: 1731278406183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:06,193 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:06,193 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:06,194 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:06,194 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:06,194 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:06,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60796 deadline: 1731278406183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:06,194 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:06,194 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:06,194 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:06,194 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:06,351 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/B/584f11543f0449d281f60660321bd01f 2024-11-10T22:39:06,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/C/526678fa872a424c8f907e1d5e2cdaca is 50, key is test_row_0/C:col10/1731278343332/Put/seqid=0 2024-11-10T22:39:06,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742181_1357 (size=12001) 2024-11-10T22:39:06,363 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/C/526678fa872a424c8f907e1d5e2cdaca 2024-11-10T22:39:06,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/A/36bc770484ee4d2983b226fcb614b6f4 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/A/36bc770484ee4d2983b226fcb614b6f4 2024-11-10T22:39:06,371 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/A/36bc770484ee4d2983b226fcb614b6f4, entries=150, sequenceid=40, filesize=30.2 K 2024-11-10T22:39:06,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/B/584f11543f0449d281f60660321bd01f as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/B/584f11543f0449d281f60660321bd01f 2024-11-10T22:39:06,377 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/B/584f11543f0449d281f60660321bd01f, entries=150, sequenceid=40, filesize=11.7 K 2024-11-10T22:39:06,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/C/526678fa872a424c8f907e1d5e2cdaca as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/C/526678fa872a424c8f907e1d5e2cdaca 2024-11-10T22:39:06,382 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/C/526678fa872a424c8f907e1d5e2cdaca, entries=150, sequenceid=40, filesize=11.7 K 2024-11-10T22:39:06,383 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 9cbf4ade36acb62156f78dbcf26aa02e in 1268ms, sequenceid=40, compaction requested=false 2024-11-10T22:39:06,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2603): Flush status journal for 9cbf4ade36acb62156f78dbcf26aa02e: 2024-11-10T22:39:06,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. 2024-11-10T22:39:06,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=106 2024-11-10T22:39:06,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43351 {}] master.HMaster(4169): Remote procedure done, pid=106 2024-11-10T22:39:06,386 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-11-10T22:39:06,386 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.1100 sec 2024-11-10T22:39:06,389 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees in 3.1160 sec 2024-11-10T22:39:06,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(8855): Flush requested on 9cbf4ade36acb62156f78dbcf26aa02e 2024-11-10T22:39:06,706 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T22:39:06,706 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9cbf4ade36acb62156f78dbcf26aa02e 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-10T22:39:06,715 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110963a9a157e5547a484cd89f2e5b2bf9f_9cbf4ade36acb62156f78dbcf26aa02e is 50, key is test_row_0/A:col10/1731278346706/Put/seqid=0 2024-11-10T22:39:06,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742182_1358 (size=17034) 2024-11-10T22:39:06,724 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:06,727 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110963a9a157e5547a484cd89f2e5b2bf9f_9cbf4ade36acb62156f78dbcf26aa02e to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110963a9a157e5547a484cd89f2e5b2bf9f_9cbf4ade36acb62156f78dbcf26aa02e 2024-11-10T22:39:06,727 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/A/a91e85b1f07c43f8b0be9bdaaf687e0f, store: [table=TestAcidGuarantees family=A region=9cbf4ade36acb62156f78dbcf26aa02e] 2024-11-10T22:39:06,728 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/A/a91e85b1f07c43f8b0be9bdaaf687e0f is 175, key is test_row_0/A:col10/1731278346706/Put/seqid=0 2024-11-10T22:39:06,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742183_1359 (size=48139) 2024-11-10T22:39:06,759 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:06,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60844 deadline: 1731278406747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:06,759 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:06,759 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:06,759 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:06,759 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:06,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60834 deadline: 1731278406748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:06,760 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:06,760 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:06,760 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:06,760 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:06,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60796 deadline: 1731278406749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:06,760 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:06,760 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:06,760 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:06,761 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:06,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60864 deadline: 1731278406749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:06,761 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:06,761 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:06,761 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:06,761 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:06,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60874 deadline: 1731278406750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:06,762 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:06,762 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:06,762 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:06,865 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:06,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60844 deadline: 1731278406863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:06,866 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:06,866 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:06,866 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:06,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60834 deadline: 1731278406863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:06,866 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:06,866 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:06,866 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:06,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60796 deadline: 1731278406863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:06,866 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:06,866 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:06,866 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:06,866 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:06,866 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:06,878 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:06,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60864 deadline: 1731278406872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:06,879 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:06,879 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:06,879 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:06,879 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:06,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60874 deadline: 1731278406872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:06,879 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:06,879 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:06,879 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:07,079 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:07,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60844 deadline: 1731278407072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:07,079 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:07,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60834 deadline: 1731278407072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:07,079 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:07,079 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:07,079 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:07,079 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:07,079 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:07,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60796 deadline: 1731278407072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:07,079 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:07,079 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:07,080 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:07,080 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:07,080 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:07,092 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:07,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60864 deadline: 1731278407082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:07,092 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:07,092 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:07,092 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:07,092 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:07,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60874 deadline: 1731278407083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:07,092 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:07,092 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:07,092 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:07,137 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=54, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/A/a91e85b1f07c43f8b0be9bdaaf687e0f 2024-11-10T22:39:07,146 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/B/3a4d01b97717458ba99b93e673b8d35c is 50, key is test_row_0/B:col10/1731278346706/Put/seqid=0 2024-11-10T22:39:07,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742184_1360 (size=12001) 2024-11-10T22:39:07,389 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:07,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60834 deadline: 1731278407382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:07,390 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:07,390 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:07,390 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:07,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60796 deadline: 1731278407382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:07,390 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:07,390 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:07,390 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:07,390 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:07,403 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:07,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60844 deadline: 1731278407392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:07,403 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:07,403 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:07,403 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:07,410 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:07,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60864 deadline: 1731278407402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:07,411 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:07,411 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:07,411 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:07,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60874 deadline: 1731278407402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:07,411 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:07,411 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:07,411 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:07,411 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:07,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-10T22:39:07,443 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T22:39:07,443 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43351 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T22:39:07,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43351 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees 2024-11-10T22:39:07,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-10T22:39:07,445 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T22:39:07,446 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T22:39:07,446 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T22:39:07,551 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=54 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/B/3a4d01b97717458ba99b93e673b8d35c 2024-11-10T22:39:07,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-10T22:39:07,558 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/C/17beedd7f8cb47f099efb2567a384f82 is 50, key is test_row_0/C:col10/1731278346706/Put/seqid=0 2024-11-10T22:39:07,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742185_1361 (size=12001) 2024-11-10T22:39:07,598 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-10T22:39:07,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. 2024-11-10T22:39:07,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. as already flushing 2024-11-10T22:39:07,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. 2024-11-10T22:39:07,599 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 9cbf4ade36acb62156f78dbcf26aa02e, NAME => 'TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:07,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 9cbf4ade36acb62156f78dbcf26aa02e, NAME => 'TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:07,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9cbf4ade36acb62156f78dbcf26aa02e, NAME => 'TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9cbf4ade36acb62156f78dbcf26aa02e, NAME => 'TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:07,751 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-10T22:39:07,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. 2024-11-10T22:39:07,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. as already flushing 2024-11-10T22:39:07,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. 2024-11-10T22:39:07,752 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 9cbf4ade36acb62156f78dbcf26aa02e, NAME => 'TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:07,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 9cbf4ade36acb62156f78dbcf26aa02e, NAME => 'TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:07,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9cbf4ade36acb62156f78dbcf26aa02e, NAME => 'TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9cbf4ade36acb62156f78dbcf26aa02e, NAME => 'TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:07,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-10T22:39:07,904 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-10T22:39:07,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. 2024-11-10T22:39:07,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. as already flushing 2024-11-10T22:39:07,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. 2024-11-10T22:39:07,905 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 9cbf4ade36acb62156f78dbcf26aa02e, NAME => 'TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:07,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 9cbf4ade36acb62156f78dbcf26aa02e, NAME => 'TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:07,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9cbf4ade36acb62156f78dbcf26aa02e, NAME => 'TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9cbf4ade36acb62156f78dbcf26aa02e, NAME => 'TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:07,912 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:07,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60834 deadline: 1731278407903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:07,912 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:07,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60796 deadline: 1731278407903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:07,912 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:07,912 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:07,912 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:07,913 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:07,913 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:07,913 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:07,916 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:07,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60844 deadline: 1731278407913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:07,917 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:07,917 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:07,917 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:07,934 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:07,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60864 deadline: 1731278407922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:07,934 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:07,934 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:07,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60874 deadline: 1731278407922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:07,934 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:07,934 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:07,935 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:07,935 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:07,935 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:07,962 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=54 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/C/17beedd7f8cb47f099efb2567a384f82 2024-11-10T22:39:07,967 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/A/a91e85b1f07c43f8b0be9bdaaf687e0f as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/A/a91e85b1f07c43f8b0be9bdaaf687e0f 2024-11-10T22:39:07,970 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/A/a91e85b1f07c43f8b0be9bdaaf687e0f, entries=250, sequenceid=54, filesize=47.0 K 2024-11-10T22:39:07,971 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/B/3a4d01b97717458ba99b93e673b8d35c as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/B/3a4d01b97717458ba99b93e673b8d35c 2024-11-10T22:39:07,976 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/B/3a4d01b97717458ba99b93e673b8d35c, entries=150, sequenceid=54, filesize=11.7 K 2024-11-10T22:39:07,977 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/C/17beedd7f8cb47f099efb2567a384f82 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/C/17beedd7f8cb47f099efb2567a384f82 2024-11-10T22:39:07,981 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/C/17beedd7f8cb47f099efb2567a384f82, entries=150, sequenceid=54, filesize=11.7 K 2024-11-10T22:39:07,982 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 9cbf4ade36acb62156f78dbcf26aa02e in 1276ms, sequenceid=54, compaction requested=true 2024-11-10T22:39:07,983 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9cbf4ade36acb62156f78dbcf26aa02e: 2024-11-10T22:39:07,983 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9cbf4ade36acb62156f78dbcf26aa02e:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T22:39:07,983 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T22:39:07,983 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9cbf4ade36acb62156f78dbcf26aa02e:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T22:39:07,983 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T22:39:07,983 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T22:39:07,983 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9cbf4ade36acb62156f78dbcf26aa02e:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T22:39:07,983 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T22:39:07,983 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T22:39:07,984 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T22:39:07,984 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 118643 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T22:39:07,984 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1541): 9cbf4ade36acb62156f78dbcf26aa02e/B is initiating minor compaction (all files) 2024-11-10T22:39:07,984 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HStore(1541): 9cbf4ade36acb62156f78dbcf26aa02e/A is initiating minor compaction (all files) 2024-11-10T22:39:07,984 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9cbf4ade36acb62156f78dbcf26aa02e/B in TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. 2024-11-10T22:39:07,984 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9cbf4ade36acb62156f78dbcf26aa02e/A in TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. 2024-11-10T22:39:07,984 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/B/df23d130d3de4aca8cd73a2c83ca92ad, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/B/584f11543f0449d281f60660321bd01f, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/B/3a4d01b97717458ba99b93e673b8d35c] into tmpdir=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp, totalSize=35.2 K 2024-11-10T22:39:07,984 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/A/ebc3917ea86d4b548ba7ba1f496355cc, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/A/36bc770484ee4d2983b226fcb614b6f4, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/A/a91e85b1f07c43f8b0be9bdaaf687e0f] into tmpdir=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp, totalSize=115.9 K 2024-11-10T22:39:07,984 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. 2024-11-10T22:39:07,984 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. files: [hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/A/ebc3917ea86d4b548ba7ba1f496355cc, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/A/36bc770484ee4d2983b226fcb614b6f4, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/A/a91e85b1f07c43f8b0be9bdaaf687e0f] 2024-11-10T22:39:07,984 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting df23d130d3de4aca8cd73a2c83ca92ad, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731278343285 2024-11-10T22:39:07,984 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting ebc3917ea86d4b548ba7ba1f496355cc, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731278343285 2024-11-10T22:39:07,985 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting 584f11543f0449d281f60660321bd01f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731278343324 2024-11-10T22:39:07,985 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 36bc770484ee4d2983b226fcb614b6f4, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731278343324 2024-11-10T22:39:07,985 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting 3a4d01b97717458ba99b93e673b8d35c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731278345518 2024-11-10T22:39:07,985 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting a91e85b1f07c43f8b0be9bdaaf687e0f, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731278345518 2024-11-10T22:39:07,991 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=9cbf4ade36acb62156f78dbcf26aa02e] 2024-11-10T22:39:07,991 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9cbf4ade36acb62156f78dbcf26aa02e#B#compaction#305 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T22:39:07,992 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/B/1062828ffdb847b985e82ee173ff9f0a is 50, key is test_row_0/B:col10/1731278346706/Put/seqid=0 2024-11-10T22:39:07,997 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241110dfc94f654eb3436da8114592798e2a9b_9cbf4ade36acb62156f78dbcf26aa02e store=[table=TestAcidGuarantees family=A region=9cbf4ade36acb62156f78dbcf26aa02e] 2024-11-10T22:39:07,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742186_1362 (size=12104) 2024-11-10T22:39:08,000 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241110dfc94f654eb3436da8114592798e2a9b_9cbf4ade36acb62156f78dbcf26aa02e, store=[table=TestAcidGuarantees family=A region=9cbf4ade36acb62156f78dbcf26aa02e] 2024-11-10T22:39:08,001 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110dfc94f654eb3436da8114592798e2a9b_9cbf4ade36acb62156f78dbcf26aa02e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=9cbf4ade36acb62156f78dbcf26aa02e] 2024-11-10T22:39:08,004 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742187_1363 (size=4469) 2024-11-10T22:39:08,058 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-10T22:39:08,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. 2024-11-10T22:39:08,058 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2902): Flushing 9cbf4ade36acb62156f78dbcf26aa02e 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-10T22:39:08,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110556fff8b927441b8a35fc0eeb50b54f0_9cbf4ade36acb62156f78dbcf26aa02e is 50, key is test_row_0/A:col10/1731278346732/Put/seqid=0 2024-11-10T22:39:08,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742188_1364 (size=12154) 2024-11-10T22:39:08,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-10T22:39:08,404 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/B/1062828ffdb847b985e82ee173ff9f0a as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/B/1062828ffdb847b985e82ee173ff9f0a 2024-11-10T22:39:08,407 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9cbf4ade36acb62156f78dbcf26aa02e#A#compaction#306 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T22:39:08,408 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/A/13260c0ea97e473eacbe152329b99485 is 175, key is test_row_0/A:col10/1731278346706/Put/seqid=0 2024-11-10T22:39:08,411 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9cbf4ade36acb62156f78dbcf26aa02e/B of 9cbf4ade36acb62156f78dbcf26aa02e into 1062828ffdb847b985e82ee173ff9f0a(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T22:39:08,412 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9cbf4ade36acb62156f78dbcf26aa02e: 2024-11-10T22:39:08,412 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., storeName=9cbf4ade36acb62156f78dbcf26aa02e/B, priority=13, startTime=1731278347983; duration=0sec 2024-11-10T22:39:08,412 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T22:39:08,412 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9cbf4ade36acb62156f78dbcf26aa02e:B 2024-11-10T22:39:08,412 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T22:39:08,413 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T22:39:08,413 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1541): 9cbf4ade36acb62156f78dbcf26aa02e/C is initiating minor compaction (all files) 2024-11-10T22:39:08,413 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9cbf4ade36acb62156f78dbcf26aa02e/C in TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. 2024-11-10T22:39:08,413 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/C/0d5ae8554bed48978fd73b35e356403c, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/C/526678fa872a424c8f907e1d5e2cdaca, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/C/17beedd7f8cb47f099efb2567a384f82] into tmpdir=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp, totalSize=35.2 K 2024-11-10T22:39:08,413 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting 0d5ae8554bed48978fd73b35e356403c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731278343285 2024-11-10T22:39:08,414 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting 526678fa872a424c8f907e1d5e2cdaca, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731278343324 2024-11-10T22:39:08,414 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting 17beedd7f8cb47f099efb2567a384f82, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731278345518 2024-11-10T22:39:08,422 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9cbf4ade36acb62156f78dbcf26aa02e#C#compaction#308 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T22:39:08,422 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/C/1c506490e59544f39155fb152e819c75 is 50, key is test_row_0/C:col10/1731278346706/Put/seqid=0 2024-11-10T22:39:08,423 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742189_1365 (size=31058) 2024-11-10T22:39:08,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742190_1366 (size=12104) 2024-11-10T22:39:08,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,474 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110556fff8b927441b8a35fc0eeb50b54f0_9cbf4ade36acb62156f78dbcf26aa02e to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110556fff8b927441b8a35fc0eeb50b54f0_9cbf4ade36acb62156f78dbcf26aa02e 2024-11-10T22:39:08,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/A/7b36acb88163429e98464f016a6453d9, store: [table=TestAcidGuarantees family=A region=9cbf4ade36acb62156f78dbcf26aa02e] 2024-11-10T22:39:08,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/A/7b36acb88163429e98464f016a6453d9 is 175, key is test_row_0/A:col10/1731278346732/Put/seqid=0 2024-11-10T22:39:08,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742191_1367 (size=30955) 2024-11-10T22:39:08,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-10T22:39:08,829 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/A/13260c0ea97e473eacbe152329b99485 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/A/13260c0ea97e473eacbe152329b99485 2024-11-10T22:39:08,833 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/C/1c506490e59544f39155fb152e819c75 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/C/1c506490e59544f39155fb152e819c75 2024-11-10T22:39:08,834 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9cbf4ade36acb62156f78dbcf26aa02e/A of 9cbf4ade36acb62156f78dbcf26aa02e into 13260c0ea97e473eacbe152329b99485(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T22:39:08,834 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9cbf4ade36acb62156f78dbcf26aa02e: 2024-11-10T22:39:08,834 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., storeName=9cbf4ade36acb62156f78dbcf26aa02e/A, priority=13, startTime=1731278347983; duration=0sec 2024-11-10T22:39:08,834 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T22:39:08,834 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9cbf4ade36acb62156f78dbcf26aa02e:A 2024-11-10T22:39:08,837 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9cbf4ade36acb62156f78dbcf26aa02e/C of 9cbf4ade36acb62156f78dbcf26aa02e into 1c506490e59544f39155fb152e819c75(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T22:39:08,837 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9cbf4ade36acb62156f78dbcf26aa02e: 2024-11-10T22:39:08,837 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., storeName=9cbf4ade36acb62156f78dbcf26aa02e/C, priority=13, startTime=1731278347983; duration=0sec 2024-11-10T22:39:08,837 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T22:39:08,837 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9cbf4ade36acb62156f78dbcf26aa02e:C 2024-11-10T22:39:08,880 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=76, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/A/7b36acb88163429e98464f016a6453d9 2024-11-10T22:39:08,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/B/6b554e8a86c346bea527af6f9cf4ad2a is 50, key is test_row_0/B:col10/1731278346732/Put/seqid=0 2024-11-10T22:39:08,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742192_1368 (size=12001) 2024-11-10T22:39:08,892 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/B/6b554e8a86c346bea527af6f9cf4ad2a 2024-11-10T22:39:08,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/C/56d35fc3349f48418d8fad098d295e23 is 50, key is test_row_0/C:col10/1731278346732/Put/seqid=0 2024-11-10T22:39:08,904 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742193_1369 (size=12001) 2024-11-10T22:39:08,904 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/C/56d35fc3349f48418d8fad098d295e23 2024-11-10T22:39:08,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/A/7b36acb88163429e98464f016a6453d9 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/A/7b36acb88163429e98464f016a6453d9 2024-11-10T22:39:08,910 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/A/7b36acb88163429e98464f016a6453d9, entries=150, sequenceid=76, filesize=30.2 K 2024-11-10T22:39:08,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/B/6b554e8a86c346bea527af6f9cf4ad2a as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/B/6b554e8a86c346bea527af6f9cf4ad2a 2024-11-10T22:39:08,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,914 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/B/6b554e8a86c346bea527af6f9cf4ad2a, entries=150, sequenceid=76, filesize=11.7 K 2024-11-10T22:39:08,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/C/56d35fc3349f48418d8fad098d295e23 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/C/56d35fc3349f48418d8fad098d295e23 2024-11-10T22:39:08,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,919 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/C/56d35fc3349f48418d8fad098d295e23, entries=150, sequenceid=76, filesize=11.7 K 2024-11-10T22:39:08,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,920 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=0 B/0 for 9cbf4ade36acb62156f78dbcf26aa02e in 862ms, sequenceid=76, compaction requested=false 2024-11-10T22:39:08,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2603): Flush status journal for 9cbf4ade36acb62156f78dbcf26aa02e: 2024-11-10T22:39:08,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. 2024-11-10T22:39:08,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=108 2024-11-10T22:39:08,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43351 {}] master.HMaster(4169): Remote procedure done, pid=108 2024-11-10T22:39:08,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,924 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-11-10T22:39:08,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,924 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4760 sec 2024-11-10T22:39:08,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,927 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees in 1.4810 sec 2024-11-10T22:39:08,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:08,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(8855): Flush requested on 9cbf4ade36acb62156f78dbcf26aa02e 2024-11-10T22:39:09,046 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T22:39:09,046 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9cbf4ade36acb62156f78dbcf26aa02e 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-10T22:39:09,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,067 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110efd1c39fbd2a448dae7eaca9b7efa630_9cbf4ade36acb62156f78dbcf26aa02e is 50, key is test_row_0/A:col10/1731278349041/Put/seqid=0 2024-11-10T22:39:09,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742195_1371 (size=24358) 2024-11-10T22:39:09,087 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,094 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110efd1c39fbd2a448dae7eaca9b7efa630_9cbf4ade36acb62156f78dbcf26aa02e to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110efd1c39fbd2a448dae7eaca9b7efa630_9cbf4ade36acb62156f78dbcf26aa02e 2024-11-10T22:39:09,095 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/A/222d356529fc407694ed7aa935f9647f, store: [table=TestAcidGuarantees family=A region=9cbf4ade36acb62156f78dbcf26aa02e] 2024-11-10T22:39:09,095 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/A/222d356529fc407694ed7aa935f9647f is 175, key is test_row_0/A:col10/1731278349041/Put/seqid=0 2024-11-10T22:39:09,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742194_1370 (size=73995) 2024-11-10T22:39:09,100 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=90, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/A/222d356529fc407694ed7aa935f9647f 2024-11-10T22:39:09,109 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/B/1ec14e8206cf42a7b8eda24f4135f0d6 is 50, key is test_row_0/B:col10/1731278349041/Put/seqid=0 2024-11-10T22:39:09,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742196_1372 (size=12001) 2024-11-10T22:39:09,123 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/B/1ec14e8206cf42a7b8eda24f4135f0d6 2024-11-10T22:39:09,136 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/C/36103108c11440fe889482b5f583f5d9 is 50, key is test_row_0/C:col10/1731278349041/Put/seqid=0 2024-11-10T22:39:09,138 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:09,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60796 deadline: 1731278409114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:09,138 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:09,138 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:09,138 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:09,138 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:09,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60834 deadline: 1731278409121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:09,139 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:09,139 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:09,139 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:09,139 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:09,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60844 deadline: 1731278409123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:09,139 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:09,139 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:09,139 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:09,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742197_1373 (size=12001) 2024-11-10T22:39:09,146 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/C/36103108c11440fe889482b5f583f5d9 2024-11-10T22:39:09,145 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:09,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60864 deadline: 1731278409134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:09,146 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:09,146 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:09,146 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:09,151 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/A/222d356529fc407694ed7aa935f9647f as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/A/222d356529fc407694ed7aa935f9647f 2024-11-10T22:39:09,154 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/A/222d356529fc407694ed7aa935f9647f, entries=400, sequenceid=90, filesize=72.3 K 2024-11-10T22:39:09,154 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/B/1ec14e8206cf42a7b8eda24f4135f0d6 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/B/1ec14e8206cf42a7b8eda24f4135f0d6 2024-11-10T22:39:09,155 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:09,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60874 deadline: 1731278409138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:09,157 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:09,157 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:09,157 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:09,163 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/B/1ec14e8206cf42a7b8eda24f4135f0d6, entries=150, sequenceid=90, filesize=11.7 K 2024-11-10T22:39:09,164 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/C/36103108c11440fe889482b5f583f5d9 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/C/36103108c11440fe889482b5f583f5d9 2024-11-10T22:39:09,170 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/C/36103108c11440fe889482b5f583f5d9, entries=150, sequenceid=90, filesize=11.7 K 2024-11-10T22:39:09,171 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 9cbf4ade36acb62156f78dbcf26aa02e in 125ms, sequenceid=90, compaction requested=true 2024-11-10T22:39:09,171 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9cbf4ade36acb62156f78dbcf26aa02e: 2024-11-10T22:39:09,171 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T22:39:09,172 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9cbf4ade36acb62156f78dbcf26aa02e:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T22:39:09,172 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T22:39:09,172 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T22:39:09,172 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9cbf4ade36acb62156f78dbcf26aa02e:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T22:39:09,172 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T22:39:09,172 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9cbf4ade36acb62156f78dbcf26aa02e:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T22:39:09,172 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T22:39:09,174 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 136008 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T22:39:09,174 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T22:39:09,174 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1541): 9cbf4ade36acb62156f78dbcf26aa02e/B is initiating minor compaction (all files) 2024-11-10T22:39:09,174 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HStore(1541): 9cbf4ade36acb62156f78dbcf26aa02e/A is initiating minor compaction (all files) 2024-11-10T22:39:09,174 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9cbf4ade36acb62156f78dbcf26aa02e/A in TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. 2024-11-10T22:39:09,174 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9cbf4ade36acb62156f78dbcf26aa02e/B in TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. 2024-11-10T22:39:09,174 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/A/13260c0ea97e473eacbe152329b99485, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/A/7b36acb88163429e98464f016a6453d9, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/A/222d356529fc407694ed7aa935f9647f] into tmpdir=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp, totalSize=132.8 K 2024-11-10T22:39:09,174 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/B/1062828ffdb847b985e82ee173ff9f0a, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/B/6b554e8a86c346bea527af6f9cf4ad2a, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/B/1ec14e8206cf42a7b8eda24f4135f0d6] into tmpdir=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp, totalSize=35.3 K 2024-11-10T22:39:09,175 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. 2024-11-10T22:39:09,175 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. files: [hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/A/13260c0ea97e473eacbe152329b99485, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/A/7b36acb88163429e98464f016a6453d9, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/A/222d356529fc407694ed7aa935f9647f] 2024-11-10T22:39:09,175 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting 1062828ffdb847b985e82ee173ff9f0a, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731278345518 2024-11-10T22:39:09,175 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 13260c0ea97e473eacbe152329b99485, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731278345518 2024-11-10T22:39:09,175 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7b36acb88163429e98464f016a6453d9, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731278346732 2024-11-10T22:39:09,175 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting 6b554e8a86c346bea527af6f9cf4ad2a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731278346732 2024-11-10T22:39:09,176 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 222d356529fc407694ed7aa935f9647f, keycount=400, bloomtype=ROW, size=72.3 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1731278349014 2024-11-10T22:39:09,176 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting 1ec14e8206cf42a7b8eda24f4135f0d6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1731278349019 2024-11-10T22:39:09,205 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=9cbf4ade36acb62156f78dbcf26aa02e] 2024-11-10T22:39:09,209 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9cbf4ade36acb62156f78dbcf26aa02e#B#compaction#314 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T22:39:09,209 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/B/b0da1e91f68743f8b9efd1e9e3278212 is 50, key is test_row_0/B:col10/1731278349041/Put/seqid=0 2024-11-10T22:39:09,216 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411100ae638b5df504672a0286eb9e56fa36a_9cbf4ade36acb62156f78dbcf26aa02e store=[table=TestAcidGuarantees family=A region=9cbf4ade36acb62156f78dbcf26aa02e] 2024-11-10T22:39:09,220 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411100ae638b5df504672a0286eb9e56fa36a_9cbf4ade36acb62156f78dbcf26aa02e, store=[table=TestAcidGuarantees family=A region=9cbf4ade36acb62156f78dbcf26aa02e] 2024-11-10T22:39:09,221 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411100ae638b5df504672a0286eb9e56fa36a_9cbf4ade36acb62156f78dbcf26aa02e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=9cbf4ade36acb62156f78dbcf26aa02e] 2024-11-10T22:39:09,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742198_1374 (size=12207) 2024-11-10T22:39:09,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742199_1375 (size=4469) 2024-11-10T22:39:09,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(8855): Flush requested on 9cbf4ade36acb62156f78dbcf26aa02e 2024-11-10T22:39:09,253 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T22:39:09,253 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9cbf4ade36acb62156f78dbcf26aa02e 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-10T22:39:09,266 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110686b162dee7f4319bdf17ecd4f19fa84_9cbf4ade36acb62156f78dbcf26aa02e is 50, key is test_row_0/A:col10/1731278349129/Put/seqid=0 2024-11-10T22:39:09,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742200_1376 (size=14594) 2024-11-10T22:39:09,274 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:09,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60874 deadline: 1731278409262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:09,275 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:09,275 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:09,275 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:09,277 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:09,281 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110686b162dee7f4319bdf17ecd4f19fa84_9cbf4ade36acb62156f78dbcf26aa02e to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110686b162dee7f4319bdf17ecd4f19fa84_9cbf4ade36acb62156f78dbcf26aa02e 2024-11-10T22:39:09,282 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/A/0e4b2856efb54e80a9242a51e30a581e, store: [table=TestAcidGuarantees family=A region=9cbf4ade36acb62156f78dbcf26aa02e] 2024-11-10T22:39:09,283 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/A/0e4b2856efb54e80a9242a51e30a581e is 175, key is test_row_0/A:col10/1731278349129/Put/seqid=0 2024-11-10T22:39:09,286 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:09,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60796 deadline: 1731278409271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:09,286 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:09,286 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:09,286 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:09,290 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:09,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60834 deadline: 1731278409273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:09,291 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:09,291 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:09,291 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:09,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60844 deadline: 1731278409273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:09,291 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:09,291 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:09,291 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:09,291 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:09,291 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:09,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60864 deadline: 1731278409276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:09,292 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:09,292 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:09,292 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:09,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742201_1377 (size=39549) 2024-11-10T22:39:09,296 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=116, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/A/0e4b2856efb54e80a9242a51e30a581e 2024-11-10T22:39:09,306 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/B/6bbe0f099df24b65b7f9c3b6854cf6fc is 50, key is test_row_0/B:col10/1731278349129/Put/seqid=0 2024-11-10T22:39:09,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742202_1378 (size=12001) 2024-11-10T22:39:09,402 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:09,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60796 deadline: 1731278409392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:09,402 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:09,402 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:09,402 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:09,412 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:09,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60834 deadline: 1731278409403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:09,412 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:09,412 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:09,412 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:09,412 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:09,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60844 deadline: 1731278409403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:09,412 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:09,412 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:09,412 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:09,412 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:09,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60864 deadline: 1731278409403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:09,413 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:09,413 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:09,413 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:09,492 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:09,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60874 deadline: 1731278409483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:09,493 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:09,493 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:09,493 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:09,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-10T22:39:09,593 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T22:39:09,594 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43351 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T22:39:09,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43351 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees 2024-11-10T22:39:09,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-10T22:39:09,596 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T22:39:09,596 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T22:39:09,596 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T22:39:09,621 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:09,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60796 deadline: 1731278409612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:09,622 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:09,622 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:09,622 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:09,635 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:09,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60834 deadline: 1731278409622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:09,636 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:09,636 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:09,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60844 deadline: 1731278409622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:09,636 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:09,636 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:09,636 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:09,636 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:09,636 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:09,636 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:09,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60864 deadline: 1731278409623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:09,636 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:09,636 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:09,636 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:09,650 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/B/b0da1e91f68743f8b9efd1e9e3278212 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/B/b0da1e91f68743f8b9efd1e9e3278212 2024-11-10T22:39:09,651 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9cbf4ade36acb62156f78dbcf26aa02e#A#compaction#315 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T22:39:09,652 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/A/74e097657c674611b18587e088c32f3f is 175, key is test_row_0/A:col10/1731278349041/Put/seqid=0 2024-11-10T22:39:09,658 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9cbf4ade36acb62156f78dbcf26aa02e/B of 9cbf4ade36acb62156f78dbcf26aa02e into b0da1e91f68743f8b9efd1e9e3278212(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T22:39:09,658 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9cbf4ade36acb62156f78dbcf26aa02e: 2024-11-10T22:39:09,658 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., storeName=9cbf4ade36acb62156f78dbcf26aa02e/B, priority=13, startTime=1731278349172; duration=0sec 2024-11-10T22:39:09,658 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742203_1379 (size=31161) 2024-11-10T22:39:09,658 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T22:39:09,658 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9cbf4ade36acb62156f78dbcf26aa02e:B 2024-11-10T22:39:09,659 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T22:39:09,660 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T22:39:09,660 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1541): 9cbf4ade36acb62156f78dbcf26aa02e/C is initiating minor compaction (all files) 2024-11-10T22:39:09,660 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9cbf4ade36acb62156f78dbcf26aa02e/C in TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. 2024-11-10T22:39:09,660 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/C/1c506490e59544f39155fb152e819c75, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/C/56d35fc3349f48418d8fad098d295e23, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/C/36103108c11440fe889482b5f583f5d9] into tmpdir=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp, totalSize=35.3 K 2024-11-10T22:39:09,660 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting 1c506490e59544f39155fb152e819c75, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731278345518 2024-11-10T22:39:09,661 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting 56d35fc3349f48418d8fad098d295e23, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731278346732 2024-11-10T22:39:09,661 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting 36103108c11440fe889482b5f583f5d9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1731278349019 2024-11-10T22:39:09,663 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/A/74e097657c674611b18587e088c32f3f as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/A/74e097657c674611b18587e088c32f3f 2024-11-10T22:39:09,669 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9cbf4ade36acb62156f78dbcf26aa02e/A of 9cbf4ade36acb62156f78dbcf26aa02e into 74e097657c674611b18587e088c32f3f(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T22:39:09,669 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9cbf4ade36acb62156f78dbcf26aa02e: 2024-11-10T22:39:09,670 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., storeName=9cbf4ade36acb62156f78dbcf26aa02e/A, priority=13, startTime=1731278349171; duration=0sec 2024-11-10T22:39:09,670 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T22:39:09,670 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9cbf4ade36acb62156f78dbcf26aa02e:A 2024-11-10T22:39:09,671 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9cbf4ade36acb62156f78dbcf26aa02e#C#compaction#318 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T22:39:09,671 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/C/4d8c0e8c655940e2b6333815e0d1fe77 is 50, key is test_row_0/C:col10/1731278349041/Put/seqid=0 2024-11-10T22:39:09,675 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742204_1380 (size=12207) 2024-11-10T22:39:09,679 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/C/4d8c0e8c655940e2b6333815e0d1fe77 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/C/4d8c0e8c655940e2b6333815e0d1fe77 2024-11-10T22:39:09,686 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9cbf4ade36acb62156f78dbcf26aa02e/C of 9cbf4ade36acb62156f78dbcf26aa02e into 4d8c0e8c655940e2b6333815e0d1fe77(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T22:39:09,686 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9cbf4ade36acb62156f78dbcf26aa02e: 2024-11-10T22:39:09,686 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., storeName=9cbf4ade36acb62156f78dbcf26aa02e/C, priority=13, startTime=1731278349172; duration=0sec 2024-11-10T22:39:09,686 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T22:39:09,686 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9cbf4ade36acb62156f78dbcf26aa02e:C 2024-11-10T22:39:09,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-10T22:39:09,712 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/B/6bbe0f099df24b65b7f9c3b6854cf6fc 2024-11-10T22:39:09,720 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/C/240a76bbd06340c0b1ca6075a1257216 is 50, key is test_row_0/C:col10/1731278349129/Put/seqid=0 2024-11-10T22:39:09,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742205_1381 (size=12001) 2024-11-10T22:39:09,748 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-10T22:39:09,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. 2024-11-10T22:39:09,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. as already flushing 2024-11-10T22:39:09,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. 2024-11-10T22:39:09,748 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 9cbf4ade36acb62156f78dbcf26aa02e, NAME => 'TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:09,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 9cbf4ade36acb62156f78dbcf26aa02e, NAME => 'TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:09,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9cbf4ade36acb62156f78dbcf26aa02e, NAME => 'TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9cbf4ade36acb62156f78dbcf26aa02e, NAME => 'TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:09,810 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:09,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60874 deadline: 1731278409802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:09,811 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:09,811 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:09,811 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:09,901 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-10T22:39:09,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. 2024-11-10T22:39:09,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. as already flushing 2024-11-10T22:39:09,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. 2024-11-10T22:39:09,902 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 9cbf4ade36acb62156f78dbcf26aa02e, NAME => 'TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:09,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 9cbf4ade36acb62156f78dbcf26aa02e, NAME => 'TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:09,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9cbf4ade36acb62156f78dbcf26aa02e, NAME => 'TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9cbf4ade36acb62156f78dbcf26aa02e, NAME => 'TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:09,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-10T22:39:09,942 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:09,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60796 deadline: 1731278409932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:09,942 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:09,942 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:09,942 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:09,951 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:09,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60834 deadline: 1731278409943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:09,952 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:09,952 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:09,952 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:09,952 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:09,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60844 deadline: 1731278409943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:09,952 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:09,952 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:09,952 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:09,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60864 deadline: 1731278409943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:09,952 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:09,952 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:09,952 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:09,952 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:10,054 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-10T22:39:10,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. 2024-11-10T22:39:10,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. as already flushing 2024-11-10T22:39:10,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. 2024-11-10T22:39:10,055 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 9cbf4ade36acb62156f78dbcf26aa02e, NAME => 'TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:10,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 9cbf4ade36acb62156f78dbcf26aa02e, NAME => 'TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:10,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9cbf4ade36acb62156f78dbcf26aa02e, NAME => 'TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9cbf4ade36acb62156f78dbcf26aa02e, NAME => 'TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:10,128 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/C/240a76bbd06340c0b1ca6075a1257216 2024-11-10T22:39:10,132 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/A/0e4b2856efb54e80a9242a51e30a581e as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/A/0e4b2856efb54e80a9242a51e30a581e 2024-11-10T22:39:10,136 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/A/0e4b2856efb54e80a9242a51e30a581e, entries=200, sequenceid=116, filesize=38.6 K 2024-11-10T22:39:10,136 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/B/6bbe0f099df24b65b7f9c3b6854cf6fc as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/B/6bbe0f099df24b65b7f9c3b6854cf6fc 2024-11-10T22:39:10,139 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/B/6bbe0f099df24b65b7f9c3b6854cf6fc, entries=150, sequenceid=116, filesize=11.7 K 2024-11-10T22:39:10,140 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/C/240a76bbd06340c0b1ca6075a1257216 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/C/240a76bbd06340c0b1ca6075a1257216 2024-11-10T22:39:10,144 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/C/240a76bbd06340c0b1ca6075a1257216, entries=150, sequenceid=116, filesize=11.7 K 2024-11-10T22:39:10,145 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 9cbf4ade36acb62156f78dbcf26aa02e in 892ms, sequenceid=116, compaction requested=false 2024-11-10T22:39:10,145 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9cbf4ade36acb62156f78dbcf26aa02e: 2024-11-10T22:39:10,207 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-10T22:39:10,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. 2024-11-10T22:39:10,208 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2902): Flushing 9cbf4ade36acb62156f78dbcf26aa02e 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-10T22:39:10,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111094e985c4a58f4ecbb131ab3f2f141502_9cbf4ade36acb62156f78dbcf26aa02e is 50, key is test_row_0/A:col10/1731278349272/Put/seqid=0 2024-11-10T22:39:10,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742206_1382 (size=12154) 2024-11-10T22:39:10,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-10T22:39:10,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(8855): Flush requested on 9cbf4ade36acb62156f78dbcf26aa02e 2024-11-10T22:39:10,328 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. as already flushing 2024-11-10T22:39:10,392 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-10T22:39:10,392 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-10T22:39:10,476 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:10,476 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:10,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60864 deadline: 1731278410463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:10,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60796 deadline: 1731278410464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:10,476 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:10,476 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:10,476 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:10,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60874 deadline: 1731278410473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:10,477 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:10,477 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:10,477 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:10,477 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:10,477 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:10,477 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:10,477 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:10,484 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:10,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60834 deadline: 1731278410475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:10,484 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:10,484 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:10,484 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:10,488 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:10,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60844 deadline: 1731278410476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:10,488 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:10,488 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:10,489 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:10,587 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:10,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60796 deadline: 1731278410583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:10,587 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:10,587 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:10,587 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:10,587 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:10,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60874 deadline: 1731278410583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:10,588 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:10,588 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:10,588 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:10,600 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:10,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60834 deadline: 1731278410592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:10,600 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:10,600 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:10,600 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:10,600 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:10,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60844 deadline: 1731278410592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:10,600 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:10,600 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:10,600 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:10,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:10,627 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111094e985c4a58f4ecbb131ab3f2f141502_9cbf4ade36acb62156f78dbcf26aa02e to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111094e985c4a58f4ecbb131ab3f2f141502_9cbf4ade36acb62156f78dbcf26aa02e 2024-11-10T22:39:10,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/A/1e10cf5907b64dc18561cbe9df3d4acd, store: [table=TestAcidGuarantees family=A region=9cbf4ade36acb62156f78dbcf26aa02e] 2024-11-10T22:39:10,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/A/1e10cf5907b64dc18561cbe9df3d4acd is 175, key is test_row_0/A:col10/1731278349272/Put/seqid=0 2024-11-10T22:39:10,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742207_1383 (size=30955) 2024-11-10T22:39:10,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-10T22:39:10,801 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:10,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60796 deadline: 1731278410792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:10,802 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:10,802 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:10,802 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:10,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60874 deadline: 1731278410792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:10,802 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:10,803 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:10,803 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:10,803 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:10,808 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:10,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60844 deadline: 1731278410802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:10,808 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:10,808 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:10,808 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:10,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60834 deadline: 1731278410802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:10,808 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:10,808 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:10,808 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:10,808 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:11,032 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=129, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/A/1e10cf5907b64dc18561cbe9df3d4acd 2024-11-10T22:39:11,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/B/5d4c589dcd524568ab44fe191b1ea5c7 is 50, key is test_row_0/B:col10/1731278349272/Put/seqid=0 2024-11-10T22:39:11,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742208_1384 (size=12001) 2024-11-10T22:39:11,119 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:11,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60796 deadline: 1731278411112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:11,119 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:11,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60874 deadline: 1731278411112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:11,119 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:11,119 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:11,119 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:11,119 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:11,119 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:11,120 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:11,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60844 deadline: 1731278411112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:11,120 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:11,120 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:11,120 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:11,120 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:11,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60834 deadline: 1731278411112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:11,120 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:11,120 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:11,120 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:11,120 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:11,443 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/B/5d4c589dcd524568ab44fe191b1ea5c7 2024-11-10T22:39:11,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/C/c81ed21ac94244cb9c6e5a65c7a889d5 is 50, key is test_row_0/C:col10/1731278349272/Put/seqid=0 2024-11-10T22:39:11,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742209_1385 (size=12001) 2024-11-10T22:39:11,490 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:11,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60864 deadline: 1731278411482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:11,491 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 849d0d71310f,43753,1731278250663 for 'test_row_1' in 9cbf4ade36acb62156f78dbcf26aa02e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2214 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-10T22:39:11,491 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:11,491 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:11,491 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:11,630 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:11,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60834 deadline: 1731278411622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:11,630 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:11,630 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:11,630 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:11,642 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:11,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60796 deadline: 1731278411632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:11,643 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:11,643 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:11,643 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:11,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60874 deadline: 1731278411633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:11,643 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:11,643 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:11,643 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:11,643 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:11,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60844 deadline: 1731278411633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:11,643 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:11,643 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:11,643 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:11,643 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:11,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-10T22:39:11,856 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/C/c81ed21ac94244cb9c6e5a65c7a889d5 2024-11-10T22:39:11,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/A/1e10cf5907b64dc18561cbe9df3d4acd as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/A/1e10cf5907b64dc18561cbe9df3d4acd 2024-11-10T22:39:11,864 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/A/1e10cf5907b64dc18561cbe9df3d4acd, entries=150, sequenceid=129, filesize=30.2 K 2024-11-10T22:39:11,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/B/5d4c589dcd524568ab44fe191b1ea5c7 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/B/5d4c589dcd524568ab44fe191b1ea5c7 2024-11-10T22:39:11,867 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/B/5d4c589dcd524568ab44fe191b1ea5c7, entries=150, sequenceid=129, filesize=11.7 K 2024-11-10T22:39:11,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/C/c81ed21ac94244cb9c6e5a65c7a889d5 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/C/c81ed21ac94244cb9c6e5a65c7a889d5 2024-11-10T22:39:11,871 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/C/c81ed21ac94244cb9c6e5a65c7a889d5, entries=150, sequenceid=129, filesize=11.7 K 2024-11-10T22:39:11,874 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=154.31 KB/158010 for 9cbf4ade36acb62156f78dbcf26aa02e in 1666ms, sequenceid=129, compaction requested=true 2024-11-10T22:39:11,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2603): Flush status journal for 9cbf4ade36acb62156f78dbcf26aa02e: 2024-11-10T22:39:11,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. 2024-11-10T22:39:11,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=110 2024-11-10T22:39:11,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43351 {}] master.HMaster(4169): Remote procedure done, pid=110 2024-11-10T22:39:11,877 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-11-10T22:39:11,878 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2800 sec 2024-11-10T22:39:11,879 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees in 2.2840 sec 2024-11-10T22:39:12,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(8855): Flush requested on 9cbf4ade36acb62156f78dbcf26aa02e 2024-11-10T22:39:12,640 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T22:39:12,640 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9cbf4ade36acb62156f78dbcf26aa02e 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-10T22:39:12,646 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110d5cf7dd339344002aa0850bce9d39e30_9cbf4ade36acb62156f78dbcf26aa02e is 50, key is test_row_0/A:col10/1731278352638/Put/seqid=0 2024-11-10T22:39:12,652 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742210_1386 (size=14794) 2024-11-10T22:39:12,677 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:12,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60834 deadline: 1731278412666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:12,678 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:12,678 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:12,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60796 deadline: 1731278412666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:12,678 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:12,678 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:12,678 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:12,678 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:12,678 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:12,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60874 deadline: 1731278412668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:12,678 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:12,679 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:12,679 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:12,679 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:12,679 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:12,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60844 deadline: 1731278412669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:12,679 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:12,679 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:12,679 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:12,784 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:12,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60834 deadline: 1731278412783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:12,785 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:12,785 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:12,785 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:12,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60796 deadline: 1731278412783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:12,785 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:12,785 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:12,785 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:12,785 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:12,785 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:12,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60874 deadline: 1731278412783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:12,786 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:12,786 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:12,786 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:12,791 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:12,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60844 deadline: 1731278412783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:12,792 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:12,792 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:12,792 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:12,999 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:12,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60834 deadline: 1731278412992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:12,999 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:13,000 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:13,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60796 deadline: 1731278412992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:13,000 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:13,000 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:13,000 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:13,000 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:13,000 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:13,000 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:13,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60874 deadline: 1731278412992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:13,000 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:13,000 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:13,000 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:13,010 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:13,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60844 deadline: 1731278413002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:13,011 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:13,011 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:13,011 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:13,059 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:13,062 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110d5cf7dd339344002aa0850bce9d39e30_9cbf4ade36acb62156f78dbcf26aa02e to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110d5cf7dd339344002aa0850bce9d39e30_9cbf4ade36acb62156f78dbcf26aa02e 2024-11-10T22:39:13,063 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/A/287461cb233043238245cf320e44ed9a, store: [table=TestAcidGuarantees family=A region=9cbf4ade36acb62156f78dbcf26aa02e] 2024-11-10T22:39:13,064 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/A/287461cb233043238245cf320e44ed9a is 175, key is test_row_0/A:col10/1731278352638/Put/seqid=0 2024-11-10T22:39:13,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742211_1387 (size=39749) 2024-11-10T22:39:13,069 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=156, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/A/287461cb233043238245cf320e44ed9a 2024-11-10T22:39:13,078 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/B/506a13e71a3b4f6888a8fdea226547da is 50, key is test_row_0/B:col10/1731278352638/Put/seqid=0 2024-11-10T22:39:13,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742212_1388 (size=12151) 2024-11-10T22:39:13,309 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:13,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60834 deadline: 1731278413302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:13,309 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:13,309 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:13,309 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:13,322 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:13,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60796 deadline: 1731278413313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:13,322 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:13,322 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:13,322 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:13,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60874 deadline: 1731278413313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:13,322 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:13,322 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:13,322 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:13,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60844 deadline: 1731278413313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:13,322 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:13,323 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:13,323 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:13,323 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:13,323 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:13,484 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/B/506a13e71a3b4f6888a8fdea226547da 2024-11-10T22:39:13,492 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/C/f20877c160a14e7fbc65d4e390093ecf is 50, key is test_row_0/C:col10/1731278352638/Put/seqid=0 2024-11-10T22:39:13,495 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742213_1389 (size=12151) 2024-11-10T22:39:13,520 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:13,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60864 deadline: 1731278413512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:13,520 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 849d0d71310f,43753,1731278250663 for 'test_row_1' in 9cbf4ade36acb62156f78dbcf26aa02e of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4244 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-10T22:39:13,521 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:13,521 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:13,521 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:13,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-10T22:39:13,773 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T22:39:13,773 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43351 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T22:39:13,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43351 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees 2024-11-10T22:39:13,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-10T22:39:13,775 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T22:39:13,776 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T22:39:13,776 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T22:39:13,818 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:13,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60834 deadline: 1731278413812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:13,819 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:13,819 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:13,819 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:13,840 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:13,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60796 deadline: 1731278413832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:13,840 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:13,840 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:13,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60844 deadline: 1731278413832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:13,840 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:13,840 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:13,840 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:13,840 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:13,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60874 deadline: 1731278413832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:13,840 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:13,840 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:13,841 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:13,841 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:13,841 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:13,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-10T22:39:13,896 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/C/f20877c160a14e7fbc65d4e390093ecf 2024-11-10T22:39:13,900 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/A/287461cb233043238245cf320e44ed9a as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/A/287461cb233043238245cf320e44ed9a 2024-11-10T22:39:13,903 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/A/287461cb233043238245cf320e44ed9a, entries=200, sequenceid=156, filesize=38.8 K 2024-11-10T22:39:13,904 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/B/506a13e71a3b4f6888a8fdea226547da as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/B/506a13e71a3b4f6888a8fdea226547da 2024-11-10T22:39:13,907 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/B/506a13e71a3b4f6888a8fdea226547da, entries=150, sequenceid=156, filesize=11.9 K 2024-11-10T22:39:13,908 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/C/f20877c160a14e7fbc65d4e390093ecf as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/C/f20877c160a14e7fbc65d4e390093ecf 2024-11-10T22:39:13,911 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/C/f20877c160a14e7fbc65d4e390093ecf, entries=150, sequenceid=156, filesize=11.9 K 2024-11-10T22:39:13,912 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for 9cbf4ade36acb62156f78dbcf26aa02e in 1272ms, sequenceid=156, compaction requested=true 2024-11-10T22:39:13,912 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9cbf4ade36acb62156f78dbcf26aa02e: 2024-11-10T22:39:13,912 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9cbf4ade36acb62156f78dbcf26aa02e:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T22:39:13,912 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T22:39:13,912 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T22:39:13,913 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T22:39:13,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9cbf4ade36acb62156f78dbcf26aa02e:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T22:39:13,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T22:39:13,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9cbf4ade36acb62156f78dbcf26aa02e:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T22:39:13,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T22:39:13,914 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 141414 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T22:39:13,914 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HStore(1541): 9cbf4ade36acb62156f78dbcf26aa02e/A is initiating minor compaction (all files) 2024-11-10T22:39:13,914 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9cbf4ade36acb62156f78dbcf26aa02e/A in TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. 2024-11-10T22:39:13,914 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/A/74e097657c674611b18587e088c32f3f, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/A/0e4b2856efb54e80a9242a51e30a581e, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/A/1e10cf5907b64dc18561cbe9df3d4acd, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/A/287461cb233043238245cf320e44ed9a] into tmpdir=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp, totalSize=138.1 K 2024-11-10T22:39:13,914 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. 2024-11-10T22:39:13,914 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. files: [hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/A/74e097657c674611b18587e088c32f3f, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/A/0e4b2856efb54e80a9242a51e30a581e, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/A/1e10cf5907b64dc18561cbe9df3d4acd, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/A/287461cb233043238245cf320e44ed9a] 2024-11-10T22:39:13,914 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48360 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T22:39:13,914 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1541): 9cbf4ade36acb62156f78dbcf26aa02e/B is initiating minor compaction (all files) 2024-11-10T22:39:13,914 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9cbf4ade36acb62156f78dbcf26aa02e/B in TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. 2024-11-10T22:39:13,914 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/B/b0da1e91f68743f8b9efd1e9e3278212, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/B/6bbe0f099df24b65b7f9c3b6854cf6fc, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/B/5d4c589dcd524568ab44fe191b1ea5c7, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/B/506a13e71a3b4f6888a8fdea226547da] into tmpdir=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp, totalSize=47.2 K 2024-11-10T22:39:13,915 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 74e097657c674611b18587e088c32f3f, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1731278349019 2024-11-10T22:39:13,915 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting b0da1e91f68743f8b9efd1e9e3278212, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1731278349019 2024-11-10T22:39:13,915 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0e4b2856efb54e80a9242a51e30a581e, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731278349119 2024-11-10T22:39:13,915 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting 6bbe0f099df24b65b7f9c3b6854cf6fc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731278349119 2024-11-10T22:39:13,915 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1e10cf5907b64dc18561cbe9df3d4acd, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1731278349269 2024-11-10T22:39:13,916 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting 5d4c589dcd524568ab44fe191b1ea5c7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1731278349269 2024-11-10T22:39:13,916 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 287461cb233043238245cf320e44ed9a, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731278350471 2024-11-10T22:39:13,916 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting 506a13e71a3b4f6888a8fdea226547da, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731278350471 2024-11-10T22:39:13,923 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=9cbf4ade36acb62156f78dbcf26aa02e] 2024-11-10T22:39:13,924 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241110afddc62dd1f34796857757c50db440ee_9cbf4ade36acb62156f78dbcf26aa02e store=[table=TestAcidGuarantees family=A region=9cbf4ade36acb62156f78dbcf26aa02e] 2024-11-10T22:39:13,925 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9cbf4ade36acb62156f78dbcf26aa02e#B#compaction#327 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T22:39:13,926 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/B/e746fc0f9b854c7c82cbbc64ebd27c25 is 50, key is test_row_0/B:col10/1731278352638/Put/seqid=0 2024-11-10T22:39:13,926 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241110afddc62dd1f34796857757c50db440ee_9cbf4ade36acb62156f78dbcf26aa02e, store=[table=TestAcidGuarantees family=A region=9cbf4ade36acb62156f78dbcf26aa02e] 2024-11-10T22:39:13,926 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110afddc62dd1f34796857757c50db440ee_9cbf4ade36acb62156f78dbcf26aa02e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=9cbf4ade36acb62156f78dbcf26aa02e] 2024-11-10T22:39:13,929 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-10T22:39:13,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. 2024-11-10T22:39:13,929 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2902): Flushing 9cbf4ade36acb62156f78dbcf26aa02e 3/3 column families, dataSize=40.25 KB heapSize=106.22 KB 2024-11-10T22:39:13,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742214_1390 (size=12493) 2024-11-10T22:39:13,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742215_1391 (size=4469) 2024-11-10T22:39:13,942 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/B/e746fc0f9b854c7c82cbbc64ebd27c25 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/B/e746fc0f9b854c7c82cbbc64ebd27c25 2024-11-10T22:39:13,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110c12631847995440dab163e4e36a27155_9cbf4ade36acb62156f78dbcf26aa02e is 50, key is test_row_0/A:col10/1731278352664/Put/seqid=0 2024-11-10T22:39:13,947 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9cbf4ade36acb62156f78dbcf26aa02e#A#compaction#326 average throughput is 1.02 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T22:39:13,948 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/A/e7645db71df9468687c0c48eaf5098f4 is 175, key is test_row_0/A:col10/1731278352638/Put/seqid=0 2024-11-10T22:39:13,950 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9cbf4ade36acb62156f78dbcf26aa02e/B of 9cbf4ade36acb62156f78dbcf26aa02e into e746fc0f9b854c7c82cbbc64ebd27c25(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T22:39:13,950 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9cbf4ade36acb62156f78dbcf26aa02e: 2024-11-10T22:39:13,950 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., storeName=9cbf4ade36acb62156f78dbcf26aa02e/B, priority=12, startTime=1731278353912; duration=0sec 2024-11-10T22:39:13,950 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T22:39:13,950 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9cbf4ade36acb62156f78dbcf26aa02e:B 2024-11-10T22:39:13,950 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T22:39:13,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742216_1392 (size=12304) 2024-11-10T22:39:13,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:13,953 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48360 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T22:39:13,953 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1541): 9cbf4ade36acb62156f78dbcf26aa02e/C is initiating minor compaction (all files) 2024-11-10T22:39:13,953 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9cbf4ade36acb62156f78dbcf26aa02e/C in TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. 2024-11-10T22:39:13,953 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/C/4d8c0e8c655940e2b6333815e0d1fe77, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/C/240a76bbd06340c0b1ca6075a1257216, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/C/c81ed21ac94244cb9c6e5a65c7a889d5, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/C/f20877c160a14e7fbc65d4e390093ecf] into tmpdir=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp, totalSize=47.2 K 2024-11-10T22:39:13,954 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting 4d8c0e8c655940e2b6333815e0d1fe77, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1731278349019 2024-11-10T22:39:13,954 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting 240a76bbd06340c0b1ca6075a1257216, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731278349119 2024-11-10T22:39:13,955 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting c81ed21ac94244cb9c6e5a65c7a889d5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1731278349269 2024-11-10T22:39:13,957 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting f20877c160a14e7fbc65d4e390093ecf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731278350471 2024-11-10T22:39:13,958 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110c12631847995440dab163e4e36a27155_9cbf4ade36acb62156f78dbcf26aa02e to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110c12631847995440dab163e4e36a27155_9cbf4ade36acb62156f78dbcf26aa02e 2024-11-10T22:39:13,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/A/f614b3b6b82041b48437a834f7cd6ae1, store: [table=TestAcidGuarantees family=A region=9cbf4ade36acb62156f78dbcf26aa02e] 2024-11-10T22:39:13,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/A/f614b3b6b82041b48437a834f7cd6ae1 is 175, key is test_row_0/A:col10/1731278352664/Put/seqid=0 2024-11-10T22:39:13,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742217_1393 (size=31447) 2024-11-10T22:39:13,974 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9cbf4ade36acb62156f78dbcf26aa02e#C#compaction#329 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T22:39:13,974 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/C/11e45ea475f942a8b5a7c68099e5acca is 50, key is test_row_0/C:col10/1731278352638/Put/seqid=0 2024-11-10T22:39:13,975 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/A/e7645db71df9468687c0c48eaf5098f4 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/A/e7645db71df9468687c0c48eaf5098f4 2024-11-10T22:39:13,984 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9cbf4ade36acb62156f78dbcf26aa02e/A of 9cbf4ade36acb62156f78dbcf26aa02e into e7645db71df9468687c0c48eaf5098f4(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T22:39:13,984 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9cbf4ade36acb62156f78dbcf26aa02e: 2024-11-10T22:39:13,984 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., storeName=9cbf4ade36acb62156f78dbcf26aa02e/A, priority=12, startTime=1731278353912; duration=0sec 2024-11-10T22:39:13,984 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T22:39:13,984 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9cbf4ade36acb62156f78dbcf26aa02e:A 2024-11-10T22:39:14,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742218_1394 (size=31105) 2024-11-10T22:39:14,014 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=165, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/A/f614b3b6b82041b48437a834f7cd6ae1 2024-11-10T22:39:14,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742219_1395 (size=12493) 2024-11-10T22:39:14,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/B/39e30713469a4135b6d9270c50f6d551 is 50, key is test_row_0/B:col10/1731278352664/Put/seqid=0 2024-11-10T22:39:14,035 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/C/11e45ea475f942a8b5a7c68099e5acca as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/C/11e45ea475f942a8b5a7c68099e5acca 2024-11-10T22:39:14,043 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9cbf4ade36acb62156f78dbcf26aa02e/C of 9cbf4ade36acb62156f78dbcf26aa02e into 11e45ea475f942a8b5a7c68099e5acca(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T22:39:14,043 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9cbf4ade36acb62156f78dbcf26aa02e: 2024-11-10T22:39:14,043 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., storeName=9cbf4ade36acb62156f78dbcf26aa02e/C, priority=12, startTime=1731278353913; duration=0sec 2024-11-10T22:39:14,043 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T22:39:14,043 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9cbf4ade36acb62156f78dbcf26aa02e:C 2024-11-10T22:39:14,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742220_1396 (size=12151) 2024-11-10T22:39:14,046 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=165 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/B/39e30713469a4135b6d9270c50f6d551 2024-11-10T22:39:14,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/C/5089b88c48ed4bd7b7456b7dec105213 is 50, key is test_row_0/C:col10/1731278352664/Put/seqid=0 2024-11-10T22:39:14,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742221_1397 (size=12151) 2024-11-10T22:39:14,069 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=165 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/C/5089b88c48ed4bd7b7456b7dec105213 2024-11-10T22:39:14,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/A/f614b3b6b82041b48437a834f7cd6ae1 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/A/f614b3b6b82041b48437a834f7cd6ae1 2024-11-10T22:39:14,078 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/A/f614b3b6b82041b48437a834f7cd6ae1, entries=150, sequenceid=165, filesize=30.4 K 2024-11-10T22:39:14,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/B/39e30713469a4135b6d9270c50f6d551 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/B/39e30713469a4135b6d9270c50f6d551 2024-11-10T22:39:14,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,084 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/B/39e30713469a4135b6d9270c50f6d551, entries=150, sequenceid=165, filesize=11.9 K 2024-11-10T22:39:14,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/C/5089b88c48ed4bd7b7456b7dec105213 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/C/5089b88c48ed4bd7b7456b7dec105213 2024-11-10T22:39:14,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,089 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/C/5089b88c48ed4bd7b7456b7dec105213, entries=150, sequenceid=165, filesize=11.9 K 2024-11-10T22:39:14,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,090 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.17 KB/108720, currentSize=0 B/0 for 9cbf4ade36acb62156f78dbcf26aa02e in 161ms, sequenceid=165, compaction requested=false 2024-11-10T22:39:14,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2603): Flush status journal for 9cbf4ade36acb62156f78dbcf26aa02e: 2024-11-10T22:39:14,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. 2024-11-10T22:39:14,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=112 2024-11-10T22:39:14,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43351 {}] master.HMaster(4169): Remote procedure done, pid=112 2024-11-10T22:39:14,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-10T22:39:14,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,094 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-11-10T22:39:14,094 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 316 msec 2024-11-10T22:39:14,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,096 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees in 321 msec 2024-11-10T22:39:14,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-10T22:39:14,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,403 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T22:39:14,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,404 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43351 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T22:39:14,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43351 {}] procedure2.ProcedureExecutor(1139): Stored pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees 2024-11-10T22:39:14,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,406 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T22:39:14,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-10T22:39:14,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,407 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T22:39:14,407 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T22:39:14,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-10T22:39:14,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,561 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-10T22:39:14,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. 2024-11-10T22:39:14,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2603): Flush status journal for 9cbf4ade36acb62156f78dbcf26aa02e: 2024-11-10T22:39:14,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. 2024-11-10T22:39:14,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=114 2024-11-10T22:39:14,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43351 {}] master.HMaster(4169): Remote procedure done, pid=114 2024-11-10T22:39:14,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,565 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-11-10T22:39:14,565 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 156 msec 2024-11-10T22:39:14,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,568 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees in 162 msec 2024-11-10T22:39:14,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-10T22:39:14,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,723 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T22:39:14,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,723 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43351 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T22:39:14,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43351 {}] procedure2.ProcedureExecutor(1139): Stored pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees 2024-11-10T22:39:14,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-10T22:39:14,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,726 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T22:39:14,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,727 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T22:39:14,727 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T22:39:14,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-10T22:39:14,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,880 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-10T22:39:14,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. 2024-11-10T22:39:14,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2603): Flush status journal for 9cbf4ade36acb62156f78dbcf26aa02e: 2024-11-10T22:39:14,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. 2024-11-10T22:39:14,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=116 2024-11-10T22:39:14,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43351 {}] master.HMaster(4169): Remote procedure done, pid=116 2024-11-10T22:39:14,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,884 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-11-10T22:39:14,884 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 155 msec 2024-11-10T22:39:14,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,886 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees in 162 msec 2024-11-10T22:39:14,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(8855): Flush requested on 9cbf4ade36acb62156f78dbcf26aa02e 2024-11-10T22:39:14,948 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T22:39:14,948 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9cbf4ade36acb62156f78dbcf26aa02e 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-10T22:39:14,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,956 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111043661318fd3a402dad846d06eb3218db_9cbf4ade36acb62156f78dbcf26aa02e is 50, key is test_row_0/A:col10/1731278354945/Put/seqid=0 2024-11-10T22:39:14,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742223_1399 (size=24758) 2024-11-10T22:39:14,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,987 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,991 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111043661318fd3a402dad846d06eb3218db_9cbf4ade36acb62156f78dbcf26aa02e to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111043661318fd3a402dad846d06eb3218db_9cbf4ade36acb62156f78dbcf26aa02e 2024-11-10T22:39:14,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,994 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/A/209ae414f2154b30b68e6a5d17bdf3f3, store: [table=TestAcidGuarantees family=A region=9cbf4ade36acb62156f78dbcf26aa02e] 2024-11-10T22:39:14,994 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/A/209ae414f2154b30b68e6a5d17bdf3f3 is 175, key is test_row_0/A:col10/1731278354945/Put/seqid=0 2024-11-10T22:39:14,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742222_1398 (size=74395) 2024-11-10T22:39:14,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,997 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=181, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/A/209ae414f2154b30b68e6a5d17bdf3f3 2024-11-10T22:39:14,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:14,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:15,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:15,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:15,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:15,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:15,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:15,005 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/B/5f0bdfa23bdd400db0af179347044626 is 50, key is test_row_0/B:col10/1731278354945/Put/seqid=0 2024-11-10T22:39:15,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:15,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:15,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:15,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:15,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:15,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:15,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:15,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:15,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:15,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:15,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:15,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:15,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:15,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:15,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:15,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:15,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:15,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:15,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742224_1400 (size=24105) 2024-11-10T22:39:15,025 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=181 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/B/5f0bdfa23bdd400db0af179347044626 2024-11-10T22:39:15,032 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/C/f9985c2b84d64198b93e7c260d5cb38a is 50, key is test_row_0/C:col10/1731278354945/Put/seqid=0 2024-11-10T22:39:15,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-10T22:39:15,043 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T22:39:15,043 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43351 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T22:39:15,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43351 {}] procedure2.ProcedureExecutor(1139): Stored pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees 2024-11-10T22:39:15,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-10T22:39:15,046 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T22:39:15,047 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T22:39:15,047 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T22:39:15,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742225_1401 (size=12151) 2024-11-10T22:39:15,062 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=181 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/C/f9985c2b84d64198b93e7c260d5cb38a 2024-11-10T22:39:15,069 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/A/209ae414f2154b30b68e6a5d17bdf3f3 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/A/209ae414f2154b30b68e6a5d17bdf3f3 2024-11-10T22:39:15,072 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/A/209ae414f2154b30b68e6a5d17bdf3f3, entries=400, sequenceid=181, filesize=72.7 K 2024-11-10T22:39:15,074 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:15,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60874 deadline: 1731278415059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:15,075 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/B/5f0bdfa23bdd400db0af179347044626 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/B/5f0bdfa23bdd400db0af179347044626 2024-11-10T22:39:15,075 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:15,075 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:15,075 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:15,079 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:15,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60844 deadline: 1731278415061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:15,079 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:15,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60834 deadline: 1731278415063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:15,079 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:15,079 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:15,079 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:15,080 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:15,080 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:15,080 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:15,083 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/B/5f0bdfa23bdd400db0af179347044626, entries=400, sequenceid=181, filesize=23.5 K 2024-11-10T22:39:15,084 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/C/f9985c2b84d64198b93e7c260d5cb38a as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/C/f9985c2b84d64198b93e7c260d5cb38a 2024-11-10T22:39:15,089 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/C/f9985c2b84d64198b93e7c260d5cb38a, entries=150, sequenceid=181, filesize=11.9 K 2024-11-10T22:39:15,089 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 9cbf4ade36acb62156f78dbcf26aa02e in 141ms, sequenceid=181, compaction requested=true 2024-11-10T22:39:15,090 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9cbf4ade36acb62156f78dbcf26aa02e: 2024-11-10T22:39:15,090 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T22:39:15,090 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9cbf4ade36acb62156f78dbcf26aa02e:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T22:39:15,090 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T22:39:15,090 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9cbf4ade36acb62156f78dbcf26aa02e:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T22:39:15,090 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T22:39:15,090 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9cbf4ade36acb62156f78dbcf26aa02e:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T22:39:15,090 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T22:39:15,090 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T22:39:15,091 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 136947 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T22:39:15,091 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HStore(1541): 9cbf4ade36acb62156f78dbcf26aa02e/A is initiating minor compaction (all files) 2024-11-10T22:39:15,091 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9cbf4ade36acb62156f78dbcf26aa02e/A in TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. 2024-11-10T22:39:15,091 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/A/e7645db71df9468687c0c48eaf5098f4, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/A/f614b3b6b82041b48437a834f7cd6ae1, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/A/209ae414f2154b30b68e6a5d17bdf3f3] into tmpdir=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp, totalSize=133.7 K 2024-11-10T22:39:15,091 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. 2024-11-10T22:39:15,091 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. files: [hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/A/e7645db71df9468687c0c48eaf5098f4, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/A/f614b3b6b82041b48437a834f7cd6ae1, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/A/209ae414f2154b30b68e6a5d17bdf3f3] 2024-11-10T22:39:15,091 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting e7645db71df9468687c0c48eaf5098f4, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731278350471 2024-11-10T22:39:15,092 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 48749 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T22:39:15,092 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1541): 9cbf4ade36acb62156f78dbcf26aa02e/B is initiating minor compaction (all files) 2024-11-10T22:39:15,092 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9cbf4ade36acb62156f78dbcf26aa02e/B in TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. 2024-11-10T22:39:15,092 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/B/e746fc0f9b854c7c82cbbc64ebd27c25, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/B/39e30713469a4135b6d9270c50f6d551, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/B/5f0bdfa23bdd400db0af179347044626] into tmpdir=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp, totalSize=47.6 K 2024-11-10T22:39:15,092 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting f614b3b6b82041b48437a834f7cd6ae1, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1731278352647 2024-11-10T22:39:15,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(8855): Flush requested on 9cbf4ade36acb62156f78dbcf26aa02e 2024-11-10T22:39:15,093 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting e746fc0f9b854c7c82cbbc64ebd27c25, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731278350471 2024-11-10T22:39:15,093 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 209ae414f2154b30b68e6a5d17bdf3f3, keycount=400, bloomtype=ROW, size=72.7 K, encoding=NONE, compression=NONE, seqNum=181, earliestPutTs=1731278354922 2024-11-10T22:39:15,093 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T22:39:15,093 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9cbf4ade36acb62156f78dbcf26aa02e 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-10T22:39:15,093 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting 39e30713469a4135b6d9270c50f6d551, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1731278352647 2024-11-10T22:39:15,094 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting 5f0bdfa23bdd400db0af179347044626, keycount=400, bloomtype=ROW, size=23.5 K, encoding=NONE, compression=NONE, seqNum=181, earliestPutTs=1731278354922 2024-11-10T22:39:15,104 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=9cbf4ade36acb62156f78dbcf26aa02e] 2024-11-10T22:39:15,110 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9cbf4ade36acb62156f78dbcf26aa02e#B#compaction#336 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T22:39:15,111 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/B/19e5de7d2a92462883bc0718c4683117 is 50, key is test_row_0/B:col10/1731278354945/Put/seqid=0 2024-11-10T22:39:15,116 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241110c4598fe68fe9449dbf634526e1d2ae53_9cbf4ade36acb62156f78dbcf26aa02e store=[table=TestAcidGuarantees family=A region=9cbf4ade36acb62156f78dbcf26aa02e] 2024-11-10T22:39:15,119 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241110c4598fe68fe9449dbf634526e1d2ae53_9cbf4ade36acb62156f78dbcf26aa02e, store=[table=TestAcidGuarantees family=A region=9cbf4ade36acb62156f78dbcf26aa02e] 2024-11-10T22:39:15,120 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110c4598fe68fe9449dbf634526e1d2ae53_9cbf4ade36acb62156f78dbcf26aa02e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=9cbf4ade36acb62156f78dbcf26aa02e] 2024-11-10T22:39:15,134 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110b68ff42e6f7c48aea5d2516c6fa52106_9cbf4ade36acb62156f78dbcf26aa02e is 50, key is test_row_0/A:col10/1731278355091/Put/seqid=0 2024-11-10T22:39:15,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742226_1402 (size=12595) 2024-11-10T22:39:15,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742227_1403 (size=4469) 2024-11-10T22:39:15,148 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/B/19e5de7d2a92462883bc0718c4683117 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/B/19e5de7d2a92462883bc0718c4683117 2024-11-10T22:39:15,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-10T22:39:15,153 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9cbf4ade36acb62156f78dbcf26aa02e/B of 9cbf4ade36acb62156f78dbcf26aa02e into 19e5de7d2a92462883bc0718c4683117(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T22:39:15,153 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9cbf4ade36acb62156f78dbcf26aa02e: 2024-11-10T22:39:15,153 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., storeName=9cbf4ade36acb62156f78dbcf26aa02e/B, priority=13, startTime=1731278355090; duration=0sec 2024-11-10T22:39:15,153 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T22:39:15,153 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9cbf4ade36acb62156f78dbcf26aa02e:B 2024-11-10T22:39:15,153 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T22:39:15,155 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T22:39:15,155 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1541): 9cbf4ade36acb62156f78dbcf26aa02e/C is initiating minor compaction (all files) 2024-11-10T22:39:15,155 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9cbf4ade36acb62156f78dbcf26aa02e/C in TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. 2024-11-10T22:39:15,155 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/C/11e45ea475f942a8b5a7c68099e5acca, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/C/5089b88c48ed4bd7b7456b7dec105213, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/C/f9985c2b84d64198b93e7c260d5cb38a] into tmpdir=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp, totalSize=35.9 K 2024-11-10T22:39:15,155 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting 11e45ea475f942a8b5a7c68099e5acca, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731278350471 2024-11-10T22:39:15,156 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting 5089b88c48ed4bd7b7456b7dec105213, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1731278352647 2024-11-10T22:39:15,156 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting f9985c2b84d64198b93e7c260d5cb38a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=181, earliestPutTs=1731278354925 2024-11-10T22:39:15,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742228_1404 (size=14794) 2024-11-10T22:39:15,159 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:15,164 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110b68ff42e6f7c48aea5d2516c6fa52106_9cbf4ade36acb62156f78dbcf26aa02e to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110b68ff42e6f7c48aea5d2516c6fa52106_9cbf4ade36acb62156f78dbcf26aa02e 2024-11-10T22:39:15,165 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/A/93a34056eca94ff2b291854df15f28bd, store: [table=TestAcidGuarantees family=A region=9cbf4ade36acb62156f78dbcf26aa02e] 2024-11-10T22:39:15,166 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/A/93a34056eca94ff2b291854df15f28bd is 175, key is test_row_0/A:col10/1731278355091/Put/seqid=0 2024-11-10T22:39:15,170 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9cbf4ade36acb62156f78dbcf26aa02e#C#compaction#338 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T22:39:15,171 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/C/337d06a60afc48d3905000ac5fb2842a is 50, key is test_row_0/C:col10/1731278354945/Put/seqid=0 2024-11-10T22:39:15,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742229_1405 (size=39749) 2024-11-10T22:39:15,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742230_1406 (size=12595) 2024-11-10T22:39:15,191 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:15,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60844 deadline: 1731278415183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:15,197 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:15,197 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:15,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60874 deadline: 1731278415183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:15,197 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:15,197 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:15,197 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:15,197 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:15,197 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:15,200 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-10T22:39:15,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. 2024-11-10T22:39:15,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. as already flushing 2024-11-10T22:39:15,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. 2024-11-10T22:39:15,200 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 9cbf4ade36acb62156f78dbcf26aa02e, NAME => 'TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:15,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 9cbf4ade36acb62156f78dbcf26aa02e, NAME => 'TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:15,200 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:15,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60834 deadline: 1731278415183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:15,201 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:15,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60796 deadline: 1731278415184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:15,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9cbf4ade36acb62156f78dbcf26aa02e, NAME => 'TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9cbf4ade36acb62156f78dbcf26aa02e, NAME => 'TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:15,202 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:15,202 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:15,202 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:15,202 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:15,203 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:15,203 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:15,319 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:15,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60796 deadline: 1731278415312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:15,319 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:15,319 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:15,319 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:15,354 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-10T22:39:15,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. 2024-11-10T22:39:15,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. as already flushing 2024-11-10T22:39:15,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. 2024-11-10T22:39:15,354 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 9cbf4ade36acb62156f78dbcf26aa02e, NAME => 'TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:15,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 9cbf4ade36acb62156f78dbcf26aa02e, NAME => 'TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:15,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9cbf4ade36acb62156f78dbcf26aa02e, NAME => 'TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9cbf4ade36acb62156f78dbcf26aa02e, NAME => 'TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:15,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-10T22:39:15,410 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:15,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60844 deadline: 1731278415402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:15,411 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:15,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60874 deadline: 1731278415402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:15,411 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:15,411 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:15,411 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:15,411 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:15,411 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:15,411 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:15,422 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:15,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60834 deadline: 1731278415413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:15,422 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:15,422 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:15,422 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:15,507 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-10T22:39:15,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. 2024-11-10T22:39:15,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. as already flushing 2024-11-10T22:39:15,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. 2024-11-10T22:39:15,508 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 9cbf4ade36acb62156f78dbcf26aa02e, NAME => 'TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:15,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 9cbf4ade36acb62156f78dbcf26aa02e, NAME => 'TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:15,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9cbf4ade36acb62156f78dbcf26aa02e, NAME => 'TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9cbf4ade36acb62156f78dbcf26aa02e, NAME => 'TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:15,530 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:15,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60796 deadline: 1731278415523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:15,530 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:15,530 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:15,530 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:15,546 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9cbf4ade36acb62156f78dbcf26aa02e#A#compaction#335 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T22:39:15,546 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/A/0166b5db2af9439a8338262bfa2a289d is 175, key is test_row_0/A:col10/1731278354945/Put/seqid=0 2024-11-10T22:39:15,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742231_1407 (size=31549) 2024-11-10T22:39:15,578 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=207, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/A/93a34056eca94ff2b291854df15f28bd 2024-11-10T22:39:15,585 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/B/e7199917fcbd49ffa940849c2d999f31 is 50, key is test_row_0/B:col10/1731278355091/Put/seqid=0 2024-11-10T22:39:15,591 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/C/337d06a60afc48d3905000ac5fb2842a as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/C/337d06a60afc48d3905000ac5fb2842a 2024-11-10T22:39:15,591 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742232_1408 (size=12151) 2024-11-10T22:39:15,595 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9cbf4ade36acb62156f78dbcf26aa02e/C of 9cbf4ade36acb62156f78dbcf26aa02e into 337d06a60afc48d3905000ac5fb2842a(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T22:39:15,595 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9cbf4ade36acb62156f78dbcf26aa02e: 2024-11-10T22:39:15,595 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., storeName=9cbf4ade36acb62156f78dbcf26aa02e/C, priority=13, startTime=1731278355090; duration=0sec 2024-11-10T22:39:15,595 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T22:39:15,595 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9cbf4ade36acb62156f78dbcf26aa02e:C 2024-11-10T22:39:15,661 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-10T22:39:15,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. 2024-11-10T22:39:15,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. as already flushing 2024-11-10T22:39:15,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. 2024-11-10T22:39:15,661 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 9cbf4ade36acb62156f78dbcf26aa02e, NAME => 'TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:15,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 9cbf4ade36acb62156f78dbcf26aa02e, NAME => 'TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:15,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9cbf4ade36acb62156f78dbcf26aa02e, NAME => 'TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9cbf4ade36acb62156f78dbcf26aa02e, NAME => 'TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:15,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-10T22:39:15,724 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:15,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60844 deadline: 1731278415722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:15,724 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:15,724 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:15,724 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:15,724 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:15,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60874 deadline: 1731278415722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:15,724 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:15,724 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:15,725 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:15,739 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:15,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60834 deadline: 1731278415732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:15,739 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:15,739 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:15,739 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:15,814 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-10T22:39:15,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. 2024-11-10T22:39:15,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. as already flushing 2024-11-10T22:39:15,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. 2024-11-10T22:39:15,814 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 9cbf4ade36acb62156f78dbcf26aa02e, NAME => 'TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:15,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 9cbf4ade36acb62156f78dbcf26aa02e, NAME => 'TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:15,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9cbf4ade36acb62156f78dbcf26aa02e, NAME => 'TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9cbf4ade36acb62156f78dbcf26aa02e, NAME => 'TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:15,842 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:15,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60796 deadline: 1731278415832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:15,843 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:15,843 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:15,843 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:15,954 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/A/0166b5db2af9439a8338262bfa2a289d as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/A/0166b5db2af9439a8338262bfa2a289d 2024-11-10T22:39:15,958 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9cbf4ade36acb62156f78dbcf26aa02e/A of 9cbf4ade36acb62156f78dbcf26aa02e into 0166b5db2af9439a8338262bfa2a289d(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T22:39:15,958 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9cbf4ade36acb62156f78dbcf26aa02e: 2024-11-10T22:39:15,958 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., storeName=9cbf4ade36acb62156f78dbcf26aa02e/A, priority=13, startTime=1731278355090; duration=0sec 2024-11-10T22:39:15,958 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T22:39:15,958 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9cbf4ade36acb62156f78dbcf26aa02e:A 2024-11-10T22:39:15,966 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-10T22:39:15,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. 2024-11-10T22:39:15,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. as already flushing 2024-11-10T22:39:15,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. 2024-11-10T22:39:15,967 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 9cbf4ade36acb62156f78dbcf26aa02e, NAME => 'TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:15,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 9cbf4ade36acb62156f78dbcf26aa02e, NAME => 'TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:15,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9cbf4ade36acb62156f78dbcf26aa02e, NAME => 'TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9cbf4ade36acb62156f78dbcf26aa02e, NAME => 'TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:15,992 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/B/e7199917fcbd49ffa940849c2d999f31 2024-11-10T22:39:15,999 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/C/dc6e32d0e3644122b747c5519db302da is 50, key is test_row_0/C:col10/1731278355091/Put/seqid=0 2024-11-10T22:39:16,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742233_1409 (size=12151) 2024-11-10T22:39:16,119 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-10T22:39:16,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. 2024-11-10T22:39:16,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. as already flushing 2024-11-10T22:39:16,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. 2024-11-10T22:39:16,120 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 9cbf4ade36acb62156f78dbcf26aa02e, NAME => 'TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:16,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 9cbf4ade36acb62156f78dbcf26aa02e, NAME => 'TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:16,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9cbf4ade36acb62156f78dbcf26aa02e, NAME => 'TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9cbf4ade36acb62156f78dbcf26aa02e, NAME => 'TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:16,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-10T22:39:16,237 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:16,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60844 deadline: 1731278416233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:16,238 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:16,238 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:16,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60874 deadline: 1731278416233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:16,238 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:16,238 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:16,238 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:16,238 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:16,238 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:16,257 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:16,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60834 deadline: 1731278416252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:16,258 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:16,258 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:16,258 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:16,273 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-10T22:39:16,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. 2024-11-10T22:39:16,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. as already flushing 2024-11-10T22:39:16,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. 2024-11-10T22:39:16,273 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 9cbf4ade36acb62156f78dbcf26aa02e, NAME => 'TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:16,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 9cbf4ade36acb62156f78dbcf26aa02e, NAME => 'TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:16,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9cbf4ade36acb62156f78dbcf26aa02e, NAME => 'TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9cbf4ade36acb62156f78dbcf26aa02e, NAME => 'TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:16,360 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:16,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60796 deadline: 1731278416353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:16,361 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:16,361 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:16,361 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:16,403 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/C/dc6e32d0e3644122b747c5519db302da 2024-11-10T22:39:16,407 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/A/93a34056eca94ff2b291854df15f28bd as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/A/93a34056eca94ff2b291854df15f28bd 2024-11-10T22:39:16,411 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/A/93a34056eca94ff2b291854df15f28bd, entries=200, sequenceid=207, filesize=38.8 K 2024-11-10T22:39:16,411 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/B/e7199917fcbd49ffa940849c2d999f31 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/B/e7199917fcbd49ffa940849c2d999f31 2024-11-10T22:39:16,415 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/B/e7199917fcbd49ffa940849c2d999f31, entries=150, sequenceid=207, filesize=11.9 K 2024-11-10T22:39:16,415 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/C/dc6e32d0e3644122b747c5519db302da as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/C/dc6e32d0e3644122b747c5519db302da 2024-11-10T22:39:16,419 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/C/dc6e32d0e3644122b747c5519db302da, entries=150, sequenceid=207, filesize=11.9 K 2024-11-10T22:39:16,420 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 9cbf4ade36acb62156f78dbcf26aa02e in 1326ms, sequenceid=207, compaction requested=false 2024-11-10T22:39:16,420 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9cbf4ade36acb62156f78dbcf26aa02e: 2024-11-10T22:39:16,426 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-10T22:39:16,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. 2024-11-10T22:39:16,426 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2902): Flushing 9cbf4ade36acb62156f78dbcf26aa02e 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-10T22:39:16,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411103c4f9d662c164b9b82635210262018b3_9cbf4ade36acb62156f78dbcf26aa02e is 50, key is test_row_0/A:col10/1731278355167/Put/seqid=0 2024-11-10T22:39:16,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742234_1410 (size=12304) 2024-11-10T22:39:16,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:16,846 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411103c4f9d662c164b9b82635210262018b3_9cbf4ade36acb62156f78dbcf26aa02e to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411103c4f9d662c164b9b82635210262018b3_9cbf4ade36acb62156f78dbcf26aa02e 2024-11-10T22:39:16,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/A/6c009ce537514052b852fc5afd829096, store: [table=TestAcidGuarantees family=A region=9cbf4ade36acb62156f78dbcf26aa02e] 2024-11-10T22:39:16,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/A/6c009ce537514052b852fc5afd829096 is 175, key is test_row_0/A:col10/1731278355167/Put/seqid=0 2024-11-10T22:39:16,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742235_1411 (size=31105) 2024-11-10T22:39:16,851 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=220, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/A/6c009ce537514052b852fc5afd829096 2024-11-10T22:39:16,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/B/6b5829f8c884488ab141cd2b6d9feaa9 is 50, key is test_row_0/B:col10/1731278355167/Put/seqid=0 2024-11-10T22:39:16,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742236_1412 (size=12151) 2024-11-10T22:39:17,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-10T22:39:17,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(8855): Flush requested on 9cbf4ade36acb62156f78dbcf26aa02e 2024-11-10T22:39:17,247 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. as already flushing 2024-11-10T22:39:17,261 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=220 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/B/6b5829f8c884488ab141cd2b6d9feaa9 2024-11-10T22:39:17,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/C/34ad85bfc0484339b92a354111e585f6 is 50, key is test_row_0/C:col10/1731278355167/Put/seqid=0 2024-11-10T22:39:17,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742237_1413 (size=12151) 2024-11-10T22:39:17,271 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=220 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/C/34ad85bfc0484339b92a354111e585f6 2024-11-10T22:39:17,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/A/6c009ce537514052b852fc5afd829096 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/A/6c009ce537514052b852fc5afd829096 2024-11-10T22:39:17,278 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/A/6c009ce537514052b852fc5afd829096, entries=150, sequenceid=220, filesize=30.4 K 2024-11-10T22:39:17,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/B/6b5829f8c884488ab141cd2b6d9feaa9 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/B/6b5829f8c884488ab141cd2b6d9feaa9 2024-11-10T22:39:17,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:17,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:17,282 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/B/6b5829f8c884488ab141cd2b6d9feaa9, entries=150, sequenceid=220, filesize=11.9 K 2024-11-10T22:39:17,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:17,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:17,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/C/34ad85bfc0484339b92a354111e585f6 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/C/34ad85bfc0484339b92a354111e585f6 2024-11-10T22:39:17,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:17,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:17,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:17,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:17,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:17,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:17,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:17,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:17,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:17,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:17,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:17,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:17,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:17,287 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/C/34ad85bfc0484339b92a354111e585f6, entries=150, sequenceid=220, filesize=11.9 K 2024-11-10T22:39:17,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:17,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:17,288 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=60.38 KB/61830 for 9cbf4ade36acb62156f78dbcf26aa02e in 861ms, sequenceid=220, compaction requested=true 2024-11-10T22:39:17,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2603): Flush status journal for 9cbf4ade36acb62156f78dbcf26aa02e: 2024-11-10T22:39:17,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. 2024-11-10T22:39:17,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=118 2024-11-10T22:39:17,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:17,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:17,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43351 {}] master.HMaster(4169): Remote procedure done, pid=118 2024-11-10T22:39:17,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:17,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:17,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:17,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(8855): Flush requested on 9cbf4ade36acb62156f78dbcf26aa02e 2024-11-10T22:39:17,290 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T22:39:17,290 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9cbf4ade36acb62156f78dbcf26aa02e 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-10T22:39:17,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:17,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:17,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:17,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:17,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:17,291 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-11-10T22:39:17,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:17,291 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2420 sec 2024-11-10T22:39:17,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:17,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:17,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:17,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:17,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:17,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:17,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:17,294 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees in 2.2480 sec 2024-11-10T22:39:17,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:17,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:17,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:17,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:17,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:17,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:17,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:17,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:17,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:17,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:17,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:17,298 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110814b3608ed0b49189b21b027b82e2b32_9cbf4ade36acb62156f78dbcf26aa02e is 50, key is test_row_0/A:col10/1731278357289/Put/seqid=0 2024-11-10T22:39:17,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:17,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:17,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:17,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:17,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:17,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:17,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:17,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:17,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:17,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:17,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:17,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:17,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:17,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:17,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:17,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:17,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:17,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:17,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:17,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:17,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:17,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:17,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:17,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:17,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:17,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:17,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:17,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:17,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:17,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:17,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:17,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:17,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:17,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:17,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:17,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:17,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:17,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:17,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:17,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:17,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:17,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:17,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:17,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:17,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:17,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:17,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742238_1414 (size=17284) 2024-11-10T22:39:17,381 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:17,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60834 deadline: 1731278417366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:17,382 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:17,382 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:17,382 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:17,385 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:17,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60796 deadline: 1731278417372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:17,386 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 849d0d71310f,43753,1731278250663 for 'test_row_1' in 9cbf4ade36acb62156f78dbcf26aa02e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2202 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-10T22:39:17,386 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:17,386 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:17,386 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:17,395 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:17,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60844 deadline: 1731278417378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:17,395 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:17,395 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:17,395 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:17,395 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:17,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60874 deadline: 1731278417379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:17,395 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:17,395 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:17,395 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:17,500 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:17,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60834 deadline: 1731278417493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:17,501 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:17,501 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:17,501 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:17,514 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:17,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60844 deadline: 1731278417503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:17,514 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:17,514 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:17,514 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:17,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60874 deadline: 1731278417503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:17,514 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:17,514 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:17,514 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:17,514 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:17,563 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:17,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60864 deadline: 1731278417552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:17,563 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 849d0d71310f,43753,1731278250663 for 'test_row_1' in 9cbf4ade36acb62156f78dbcf26aa02e of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8287 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-10T22:39:17,564 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:17,564 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:17,564 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:17,718 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:17,722 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110814b3608ed0b49189b21b027b82e2b32_9cbf4ade36acb62156f78dbcf26aa02e to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110814b3608ed0b49189b21b027b82e2b32_9cbf4ade36acb62156f78dbcf26aa02e 2024-11-10T22:39:17,723 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/A/f5714406dbc0475bae0919cbc261bbae, store: [table=TestAcidGuarantees family=A region=9cbf4ade36acb62156f78dbcf26aa02e] 2024-11-10T22:39:17,723 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:17,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60834 deadline: 1731278417713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:17,723 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/A/f5714406dbc0475bae0919cbc261bbae is 175, key is test_row_0/A:col10/1731278357289/Put/seqid=0 2024-11-10T22:39:17,724 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:17,724 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:17,724 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:17,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742239_1415 (size=48389) 2024-11-10T22:39:17,732 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:17,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60874 deadline: 1731278417722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:17,733 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:17,733 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:17,733 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:17,733 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:17,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60844 deadline: 1731278417722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:17,733 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:17,733 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:17,733 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:18,043 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:18,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60834 deadline: 1731278418032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:18,043 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:18,043 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:18,043 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:18,051 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:18,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60874 deadline: 1731278418043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:18,052 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:18,052 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:18,052 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:18,052 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:18,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60844 deadline: 1731278418043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:18,052 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:18,052 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:18,052 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:18,129 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=233, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/A/f5714406dbc0475bae0919cbc261bbae 2024-11-10T22:39:18,136 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/B/de2c518c7d3b410baf03fd42bc6e624a is 50, key is test_row_0/B:col10/1731278357289/Put/seqid=0 2024-11-10T22:39:18,139 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742240_1416 (size=12151) 2024-11-10T22:39:18,540 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/B/de2c518c7d3b410baf03fd42bc6e624a 2024-11-10T22:39:18,548 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/C/74e04cdc74d0484ab4857d96ff51726a is 50, key is test_row_0/C:col10/1731278357289/Put/seqid=0 2024-11-10T22:39:18,551 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742241_1417 (size=12151) 2024-11-10T22:39:18,560 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:18,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60834 deadline: 1731278418552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:18,560 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:18,560 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:18,560 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:18,573 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:18,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60874 deadline: 1731278418563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:18,573 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:18,573 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:18,573 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:18,573 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:18,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60844 deadline: 1731278418563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:18,574 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:18,574 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:18,574 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:18,952 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/C/74e04cdc74d0484ab4857d96ff51726a 2024-11-10T22:39:18,956 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/A/f5714406dbc0475bae0919cbc261bbae as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/A/f5714406dbc0475bae0919cbc261bbae 2024-11-10T22:39:18,959 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/A/f5714406dbc0475bae0919cbc261bbae, entries=250, sequenceid=233, filesize=47.3 K 2024-11-10T22:39:18,960 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/B/de2c518c7d3b410baf03fd42bc6e624a as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/B/de2c518c7d3b410baf03fd42bc6e624a 2024-11-10T22:39:18,963 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/B/de2c518c7d3b410baf03fd42bc6e624a, entries=150, sequenceid=233, filesize=11.9 K 2024-11-10T22:39:18,964 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/C/74e04cdc74d0484ab4857d96ff51726a as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/C/74e04cdc74d0484ab4857d96ff51726a 2024-11-10T22:39:18,967 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/C/74e04cdc74d0484ab4857d96ff51726a, entries=150, sequenceid=233, filesize=11.9 K 2024-11-10T22:39:18,968 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 9cbf4ade36acb62156f78dbcf26aa02e in 1678ms, sequenceid=233, compaction requested=true 2024-11-10T22:39:18,968 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9cbf4ade36acb62156f78dbcf26aa02e: 2024-11-10T22:39:18,968 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9cbf4ade36acb62156f78dbcf26aa02e:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T22:39:18,968 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T22:39:18,968 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T22:39:18,968 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9cbf4ade36acb62156f78dbcf26aa02e:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T22:39:18,968 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T22:39:18,968 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9cbf4ade36acb62156f78dbcf26aa02e:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T22:39:18,968 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T22:39:18,968 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T22:39:18,969 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 150792 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T22:39:18,969 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49048 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T22:39:18,969 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1541): 9cbf4ade36acb62156f78dbcf26aa02e/B is initiating minor compaction (all files) 2024-11-10T22:39:18,969 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HStore(1541): 9cbf4ade36acb62156f78dbcf26aa02e/A is initiating minor compaction (all files) 2024-11-10T22:39:18,969 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9cbf4ade36acb62156f78dbcf26aa02e/B in TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. 2024-11-10T22:39:18,969 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9cbf4ade36acb62156f78dbcf26aa02e/A in TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. 2024-11-10T22:39:18,969 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/B/19e5de7d2a92462883bc0718c4683117, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/B/e7199917fcbd49ffa940849c2d999f31, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/B/6b5829f8c884488ab141cd2b6d9feaa9, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/B/de2c518c7d3b410baf03fd42bc6e624a] into tmpdir=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp, totalSize=47.9 K 2024-11-10T22:39:18,969 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/A/0166b5db2af9439a8338262bfa2a289d, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/A/93a34056eca94ff2b291854df15f28bd, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/A/6c009ce537514052b852fc5afd829096, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/A/f5714406dbc0475bae0919cbc261bbae] into tmpdir=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp, totalSize=147.3 K 2024-11-10T22:39:18,969 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. 2024-11-10T22:39:18,970 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. files: [hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/A/0166b5db2af9439a8338262bfa2a289d, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/A/93a34056eca94ff2b291854df15f28bd, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/A/6c009ce537514052b852fc5afd829096, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/A/f5714406dbc0475bae0919cbc261bbae] 2024-11-10T22:39:18,970 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0166b5db2af9439a8338262bfa2a289d, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=181, earliestPutTs=1731278354925 2024-11-10T22:39:18,970 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting 19e5de7d2a92462883bc0718c4683117, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=181, earliestPutTs=1731278354925 2024-11-10T22:39:18,970 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 93a34056eca94ff2b291854df15f28bd, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1731278355057 2024-11-10T22:39:18,971 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6c009ce537514052b852fc5afd829096, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1731278355152 2024-11-10T22:39:18,971 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting f5714406dbc0475bae0919cbc261bbae, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731278357268 2024-11-10T22:39:18,971 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting e7199917fcbd49ffa940849c2d999f31, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1731278355057 2024-11-10T22:39:18,971 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting 6b5829f8c884488ab141cd2b6d9feaa9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1731278355152 2024-11-10T22:39:18,971 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting de2c518c7d3b410baf03fd42bc6e624a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731278357279 2024-11-10T22:39:18,977 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=9cbf4ade36acb62156f78dbcf26aa02e] 2024-11-10T22:39:18,980 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9cbf4ade36acb62156f78dbcf26aa02e#B#compaction#348 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T22:39:18,980 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111049325d8470c74b468479b7354afbb25d_9cbf4ade36acb62156f78dbcf26aa02e store=[table=TestAcidGuarantees family=A region=9cbf4ade36acb62156f78dbcf26aa02e] 2024-11-10T22:39:18,980 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/B/b1e8b376d276484eb51e8f181ed1b1e0 is 50, key is test_row_0/B:col10/1731278357289/Put/seqid=0 2024-11-10T22:39:18,983 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111049325d8470c74b468479b7354afbb25d_9cbf4ade36acb62156f78dbcf26aa02e, store=[table=TestAcidGuarantees family=A region=9cbf4ade36acb62156f78dbcf26aa02e] 2024-11-10T22:39:18,983 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111049325d8470c74b468479b7354afbb25d_9cbf4ade36acb62156f78dbcf26aa02e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=9cbf4ade36acb62156f78dbcf26aa02e] 2024-11-10T22:39:18,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742242_1418 (size=12731) 2024-11-10T22:39:18,987 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742243_1419 (size=4469) 2024-11-10T22:39:19,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-10T22:39:19,213 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T22:39:19,214 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43351 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T22:39:19,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43351 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees 2024-11-10T22:39:19,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-10T22:39:19,216 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T22:39:19,216 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T22:39:19,216 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T22:39:19,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-10T22:39:19,369 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-10T22:39:19,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. 2024-11-10T22:39:19,369 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2902): Flushing 9cbf4ade36acb62156f78dbcf26aa02e 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-10T22:39:19,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110309b11d1f0a14ecc9d278c0f81d9bfb7_9cbf4ade36acb62156f78dbcf26aa02e is 50, key is test_row_0/A:col10/1731278357376/Put/seqid=0 2024-11-10T22:39:19,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742244_1420 (size=12304) 2024-11-10T22:39:19,388 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9cbf4ade36acb62156f78dbcf26aa02e#A#compaction#347 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T22:39:19,389 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/A/fa55a8ad318e4811accba285c0530ad1 is 175, key is test_row_0/A:col10/1731278357289/Put/seqid=0 2024-11-10T22:39:19,390 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/B/b1e8b376d276484eb51e8f181ed1b1e0 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/B/b1e8b376d276484eb51e8f181ed1b1e0 2024-11-10T22:39:19,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742245_1421 (size=31685) 2024-11-10T22:39:19,395 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9cbf4ade36acb62156f78dbcf26aa02e/B of 9cbf4ade36acb62156f78dbcf26aa02e into b1e8b376d276484eb51e8f181ed1b1e0(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T22:39:19,395 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9cbf4ade36acb62156f78dbcf26aa02e: 2024-11-10T22:39:19,395 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., storeName=9cbf4ade36acb62156f78dbcf26aa02e/B, priority=12, startTime=1731278358968; duration=0sec 2024-11-10T22:39:19,395 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T22:39:19,395 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9cbf4ade36acb62156f78dbcf26aa02e:B 2024-11-10T22:39:19,395 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T22:39:19,396 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/A/fa55a8ad318e4811accba285c0530ad1 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/A/fa55a8ad318e4811accba285c0530ad1 2024-11-10T22:39:19,396 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49048 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T22:39:19,396 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1541): 9cbf4ade36acb62156f78dbcf26aa02e/C is initiating minor compaction (all files) 2024-11-10T22:39:19,397 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9cbf4ade36acb62156f78dbcf26aa02e/C in TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. 2024-11-10T22:39:19,397 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/C/337d06a60afc48d3905000ac5fb2842a, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/C/dc6e32d0e3644122b747c5519db302da, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/C/34ad85bfc0484339b92a354111e585f6, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/C/74e04cdc74d0484ab4857d96ff51726a] into tmpdir=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp, totalSize=47.9 K 2024-11-10T22:39:19,397 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting 337d06a60afc48d3905000ac5fb2842a, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=181, earliestPutTs=1731278354925 2024-11-10T22:39:19,398 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting dc6e32d0e3644122b747c5519db302da, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1731278355057 2024-11-10T22:39:19,398 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting 34ad85bfc0484339b92a354111e585f6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1731278355152 2024-11-10T22:39:19,399 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting 74e04cdc74d0484ab4857d96ff51726a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731278357279 2024-11-10T22:39:19,402 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9cbf4ade36acb62156f78dbcf26aa02e/A of 9cbf4ade36acb62156f78dbcf26aa02e into fa55a8ad318e4811accba285c0530ad1(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T22:39:19,402 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9cbf4ade36acb62156f78dbcf26aa02e: 2024-11-10T22:39:19,402 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., storeName=9cbf4ade36acb62156f78dbcf26aa02e/A, priority=12, startTime=1731278358968; duration=0sec 2024-11-10T22:39:19,402 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T22:39:19,402 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9cbf4ade36acb62156f78dbcf26aa02e:A 2024-11-10T22:39:19,411 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9cbf4ade36acb62156f78dbcf26aa02e#C#compaction#350 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T22:39:19,411 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/C/7729a1aae8a84965b3ec78898a6f096e is 50, key is test_row_0/C:col10/1731278357289/Put/seqid=0 2024-11-10T22:39:19,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(8855): Flush requested on 9cbf4ade36acb62156f78dbcf26aa02e 2024-11-10T22:39:19,416 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. as already flushing 2024-11-10T22:39:19,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742246_1422 (size=12731) 2024-11-10T22:39:19,425 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/C/7729a1aae8a84965b3ec78898a6f096e as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/C/7729a1aae8a84965b3ec78898a6f096e 2024-11-10T22:39:19,429 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9cbf4ade36acb62156f78dbcf26aa02e/C of 9cbf4ade36acb62156f78dbcf26aa02e into 7729a1aae8a84965b3ec78898a6f096e(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T22:39:19,429 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9cbf4ade36acb62156f78dbcf26aa02e: 2024-11-10T22:39:19,429 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., storeName=9cbf4ade36acb62156f78dbcf26aa02e/C, priority=12, startTime=1731278358968; duration=0sec 2024-11-10T22:39:19,429 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T22:39:19,429 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9cbf4ade36acb62156f78dbcf26aa02e:C 2024-11-10T22:39:19,477 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:19,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60796 deadline: 1731278419470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:19,477 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:19,478 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:19,478 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:19,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-10T22:39:19,567 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:19,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60834 deadline: 1731278419562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:19,568 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 849d0d71310f,43753,1731278250663 for 'test_row_1' in 9cbf4ade36acb62156f78dbcf26aa02e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2201 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-10T22:39:19,569 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:19,569 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:19,569 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:19,589 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:19,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60796 deadline: 1731278419582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:19,589 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:19,589 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:19,589 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:19,600 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:19,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60874 deadline: 1731278419592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:19,600 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:19,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60844 deadline: 1731278419592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:19,600 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 849d0d71310f,43753,1731278250663 for 'test_row_0' in 9cbf4ade36acb62156f78dbcf26aa02e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2221 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-10T22:39:19,601 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:19,601 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:19,601 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:19,601 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 849d0d71310f,43753,1731278250663 for 'test_row_1' in 9cbf4ade36acb62156f78dbcf26aa02e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2222 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-10T22:39:19,601 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:19,601 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:19,601 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:19,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:19,785 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110309b11d1f0a14ecc9d278c0f81d9bfb7_9cbf4ade36acb62156f78dbcf26aa02e to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110309b11d1f0a14ecc9d278c0f81d9bfb7_9cbf4ade36acb62156f78dbcf26aa02e 2024-11-10T22:39:19,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/A/5973fccc77d7405f8a0ee8dcaeb151e5, store: [table=TestAcidGuarantees family=A region=9cbf4ade36acb62156f78dbcf26aa02e] 2024-11-10T22:39:19,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/A/5973fccc77d7405f8a0ee8dcaeb151e5 is 175, key is test_row_0/A:col10/1731278357376/Put/seqid=0 2024-11-10T22:39:19,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742247_1423 (size=31105) 2024-11-10T22:39:19,791 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=256, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/A/5973fccc77d7405f8a0ee8dcaeb151e5 2024-11-10T22:39:19,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/B/123f1b13f83a481cb022a97fab82bca0 is 50, key is test_row_0/B:col10/1731278357376/Put/seqid=0 2024-11-10T22:39:19,802 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:19,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60796 deadline: 1731278419792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:19,802 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:19,802 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:19,802 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:19,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742248_1424 (size=12151) 2024-11-10T22:39:19,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-10T22:39:20,120 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:20,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60796 deadline: 1731278420112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:20,120 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:20,120 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:20,120 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:20,203 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=256 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/B/123f1b13f83a481cb022a97fab82bca0 2024-11-10T22:39:20,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/C/e27d134d41d349e58087dff558a29b98 is 50, key is test_row_0/C:col10/1731278357376/Put/seqid=0 2024-11-10T22:39:20,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742249_1425 (size=12151) 2024-11-10T22:39:20,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-10T22:39:20,614 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=256 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/C/e27d134d41d349e58087dff558a29b98 2024-11-10T22:39:20,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/A/5973fccc77d7405f8a0ee8dcaeb151e5 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/A/5973fccc77d7405f8a0ee8dcaeb151e5 2024-11-10T22:39:20,623 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/A/5973fccc77d7405f8a0ee8dcaeb151e5, entries=150, sequenceid=256, filesize=30.4 K 2024-11-10T22:39:20,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/B/123f1b13f83a481cb022a97fab82bca0 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/B/123f1b13f83a481cb022a97fab82bca0 2024-11-10T22:39:20,630 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/B/123f1b13f83a481cb022a97fab82bca0, entries=150, sequenceid=256, filesize=11.9 K 2024-11-10T22:39:20,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/C/e27d134d41d349e58087dff558a29b98 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/C/e27d134d41d349e58087dff558a29b98 2024-11-10T22:39:20,635 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/C/e27d134d41d349e58087dff558a29b98, entries=150, sequenceid=256, filesize=11.9 K 2024-11-10T22:39:20,636 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 9cbf4ade36acb62156f78dbcf26aa02e in 1267ms, sequenceid=256, compaction requested=false 2024-11-10T22:39:20,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2603): Flush status journal for 9cbf4ade36acb62156f78dbcf26aa02e: 2024-11-10T22:39:20,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. 2024-11-10T22:39:20,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=120 2024-11-10T22:39:20,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43351 {}] master.HMaster(4169): Remote procedure done, pid=120 2024-11-10T22:39:20,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(8855): Flush requested on 9cbf4ade36acb62156f78dbcf26aa02e 2024-11-10T22:39:20,638 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T22:39:20,638 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9cbf4ade36acb62156f78dbcf26aa02e 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-10T22:39:20,642 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-11-10T22:39:20,642 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4220 sec 2024-11-10T22:39:20,644 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees in 1.4280 sec 2024-11-10T22:39:20,648 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411102d26124b9b6c4083a68a8f791bbbc17c_9cbf4ade36acb62156f78dbcf26aa02e is 50, key is test_row_0/A:col10/1731278359462/Put/seqid=0 2024-11-10T22:39:20,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742250_1426 (size=12454) 2024-11-10T22:39:20,657 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:20,665 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411102d26124b9b6c4083a68a8f791bbbc17c_9cbf4ade36acb62156f78dbcf26aa02e to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411102d26124b9b6c4083a68a8f791bbbc17c_9cbf4ade36acb62156f78dbcf26aa02e 2024-11-10T22:39:20,666 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/A/52b01f937b514165b9bad76d49ffc2fe, store: [table=TestAcidGuarantees family=A region=9cbf4ade36acb62156f78dbcf26aa02e] 2024-11-10T22:39:20,667 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/A/52b01f937b514165b9bad76d49ffc2fe is 175, key is test_row_0/A:col10/1731278359462/Put/seqid=0 2024-11-10T22:39:20,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742251_1427 (size=31255) 2024-11-10T22:39:20,817 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:20,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60796 deadline: 1731278420807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:20,817 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:20,817 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:20,818 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:20,928 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:20,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60796 deadline: 1731278420922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:20,929 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:20,929 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:20,929 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:21,074 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=273, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/A/52b01f937b514165b9bad76d49ffc2fe 2024-11-10T22:39:21,081 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/B/c47556c842d945c7bf1add7de697f6b2 is 50, key is test_row_0/B:col10/1731278359462/Put/seqid=0 2024-11-10T22:39:21,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742252_1428 (size=12301) 2024-11-10T22:39:21,140 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:21,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60796 deadline: 1731278421132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:21,140 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:21,140 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:21,141 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:21,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-10T22:39:21,373 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T22:39:21,373 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43351 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T22:39:21,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43351 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees 2024-11-10T22:39:21,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-10T22:39:21,375 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T22:39:21,376 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T22:39:21,376 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T22:39:21,458 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:21,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60796 deadline: 1731278421452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:21,459 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:21,459 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:21,459 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:21,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-10T22:39:21,484 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=273 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/B/c47556c842d945c7bf1add7de697f6b2 2024-11-10T22:39:21,490 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/C/1f45420e2bae436b8f1affbf32b91a3d is 50, key is test_row_0/C:col10/1731278359462/Put/seqid=0 2024-11-10T22:39:21,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742253_1429 (size=12301) 2024-11-10T22:39:21,528 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-10T22:39:21,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. 2024-11-10T22:39:21,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. as already flushing 2024-11-10T22:39:21,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. 2024-11-10T22:39:21,529 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 9cbf4ade36acb62156f78dbcf26aa02e, NAME => 'TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:21,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 9cbf4ade36acb62156f78dbcf26aa02e, NAME => 'TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:21,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9cbf4ade36acb62156f78dbcf26aa02e, NAME => 'TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9cbf4ade36acb62156f78dbcf26aa02e, NAME => 'TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:21,587 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:21,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60834 deadline: 1731278421582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:21,587 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 849d0d71310f,43753,1731278250663 for 'test_row_1' in 9cbf4ade36acb62156f78dbcf26aa02e of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4221 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-10T22:39:21,588 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:21,588 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:21,588 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:21,622 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:21,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60844 deadline: 1731278421613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:21,623 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 849d0d71310f,43753,1731278250663 for 'test_row_1' in 9cbf4ade36acb62156f78dbcf26aa02e of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4245 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-10T22:39:21,623 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:21,623 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:21,623 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:21,627 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:21,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60874 deadline: 1731278421622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:21,628 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 849d0d71310f,43753,1731278250663 for 'test_row_0' in 9cbf4ade36acb62156f78dbcf26aa02e of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4248 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-10T22:39:21,628 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:21,628 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:21,628 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:21,681 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-10T22:39:21,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. 2024-11-10T22:39:21,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. as already flushing 2024-11-10T22:39:21,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. 2024-11-10T22:39:21,682 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 9cbf4ade36acb62156f78dbcf26aa02e, NAME => 'TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:21,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 9cbf4ade36acb62156f78dbcf26aa02e, NAME => 'TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:21,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9cbf4ade36acb62156f78dbcf26aa02e, NAME => 'TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9cbf4ade36acb62156f78dbcf26aa02e, NAME => 'TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:21,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-10T22:39:21,835 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-10T22:39:21,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. 2024-11-10T22:39:21,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. as already flushing 2024-11-10T22:39:21,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. 2024-11-10T22:39:21,835 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 9cbf4ade36acb62156f78dbcf26aa02e, NAME => 'TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:21,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 9cbf4ade36acb62156f78dbcf26aa02e, NAME => 'TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:21,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9cbf4ade36acb62156f78dbcf26aa02e, NAME => 'TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9cbf4ade36acb62156f78dbcf26aa02e, NAME => 'TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:21,894 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=273 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/C/1f45420e2bae436b8f1affbf32b91a3d 2024-11-10T22:39:21,898 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/A/52b01f937b514165b9bad76d49ffc2fe as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/A/52b01f937b514165b9bad76d49ffc2fe 2024-11-10T22:39:21,902 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/A/52b01f937b514165b9bad76d49ffc2fe, entries=150, sequenceid=273, filesize=30.5 K 2024-11-10T22:39:21,903 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/B/c47556c842d945c7bf1add7de697f6b2 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/B/c47556c842d945c7bf1add7de697f6b2 2024-11-10T22:39:21,906 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/B/c47556c842d945c7bf1add7de697f6b2, entries=150, sequenceid=273, filesize=12.0 K 2024-11-10T22:39:21,907 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/C/1f45420e2bae436b8f1affbf32b91a3d as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/C/1f45420e2bae436b8f1affbf32b91a3d 2024-11-10T22:39:21,910 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/C/1f45420e2bae436b8f1affbf32b91a3d, entries=150, sequenceid=273, filesize=12.0 K 2024-11-10T22:39:21,911 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 9cbf4ade36acb62156f78dbcf26aa02e in 1273ms, sequenceid=273, compaction requested=true 2024-11-10T22:39:21,911 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9cbf4ade36acb62156f78dbcf26aa02e: 2024-11-10T22:39:21,911 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9cbf4ade36acb62156f78dbcf26aa02e:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T22:39:21,911 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T22:39:21,911 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9cbf4ade36acb62156f78dbcf26aa02e:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T22:39:21,911 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T22:39:21,912 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9cbf4ade36acb62156f78dbcf26aa02e:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T22:39:21,912 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-10T22:39:21,912 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T22:39:21,912 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T22:39:21,912 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94045 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T22:39:21,912 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HStore(1541): 9cbf4ade36acb62156f78dbcf26aa02e/A is initiating minor compaction (all files) 2024-11-10T22:39:21,913 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9cbf4ade36acb62156f78dbcf26aa02e/A in TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. 2024-11-10T22:39:21,913 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/A/fa55a8ad318e4811accba285c0530ad1, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/A/5973fccc77d7405f8a0ee8dcaeb151e5, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/A/52b01f937b514165b9bad76d49ffc2fe] into tmpdir=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp, totalSize=91.8 K 2024-11-10T22:39:21,913 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. 2024-11-10T22:39:21,913 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. files: [hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/A/fa55a8ad318e4811accba285c0530ad1, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/A/5973fccc77d7405f8a0ee8dcaeb151e5, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/A/52b01f937b514165b9bad76d49ffc2fe] 2024-11-10T22:39:21,913 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37183 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T22:39:21,913 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1541): 9cbf4ade36acb62156f78dbcf26aa02e/B is initiating minor compaction (all files) 2024-11-10T22:39:21,913 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9cbf4ade36acb62156f78dbcf26aa02e/B in TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. 2024-11-10T22:39:21,913 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting fa55a8ad318e4811accba285c0530ad1, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731278357279 2024-11-10T22:39:21,913 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/B/b1e8b376d276484eb51e8f181ed1b1e0, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/B/123f1b13f83a481cb022a97fab82bca0, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/B/c47556c842d945c7bf1add7de697f6b2] into tmpdir=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp, totalSize=36.3 K 2024-11-10T22:39:21,913 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting b1e8b376d276484eb51e8f181ed1b1e0, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731278357279 2024-11-10T22:39:21,913 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5973fccc77d7405f8a0ee8dcaeb151e5, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1731278357365 2024-11-10T22:39:21,914 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting 123f1b13f83a481cb022a97fab82bca0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1731278357365 2024-11-10T22:39:21,914 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 52b01f937b514165b9bad76d49ffc2fe, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1731278359450 2024-11-10T22:39:21,914 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting c47556c842d945c7bf1add7de697f6b2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1731278359450 2024-11-10T22:39:21,920 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=9cbf4ade36acb62156f78dbcf26aa02e] 2024-11-10T22:39:21,922 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9cbf4ade36acb62156f78dbcf26aa02e#B#compaction#356 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T22:39:21,922 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/B/f18d368743504cdc960c18153b8e9319 is 50, key is test_row_0/B:col10/1731278359462/Put/seqid=0 2024-11-10T22:39:21,924 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411108a3d2d700a064b8a9ab3c3bc71471f7e_9cbf4ade36acb62156f78dbcf26aa02e store=[table=TestAcidGuarantees family=A region=9cbf4ade36acb62156f78dbcf26aa02e] 2024-11-10T22:39:21,925 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411108a3d2d700a064b8a9ab3c3bc71471f7e_9cbf4ade36acb62156f78dbcf26aa02e, store=[table=TestAcidGuarantees family=A region=9cbf4ade36acb62156f78dbcf26aa02e] 2024-11-10T22:39:21,925 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411108a3d2d700a064b8a9ab3c3bc71471f7e_9cbf4ade36acb62156f78dbcf26aa02e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=9cbf4ade36acb62156f78dbcf26aa02e] 2024-11-10T22:39:21,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742254_1430 (size=12983) 2024-11-10T22:39:21,936 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/B/f18d368743504cdc960c18153b8e9319 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/B/f18d368743504cdc960c18153b8e9319 2024-11-10T22:39:21,937 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742255_1431 (size=4469) 2024-11-10T22:39:21,940 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9cbf4ade36acb62156f78dbcf26aa02e#A#compaction#357 average throughput is 1.22 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T22:39:21,940 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/A/0bb00ed892114513b0b8680f90fcdd00 is 175, key is test_row_0/A:col10/1731278359462/Put/seqid=0 2024-11-10T22:39:21,943 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9cbf4ade36acb62156f78dbcf26aa02e/B of 9cbf4ade36acb62156f78dbcf26aa02e into f18d368743504cdc960c18153b8e9319(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T22:39:21,943 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9cbf4ade36acb62156f78dbcf26aa02e: 2024-11-10T22:39:21,943 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., storeName=9cbf4ade36acb62156f78dbcf26aa02e/B, priority=13, startTime=1731278361911; duration=0sec 2024-11-10T22:39:21,943 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T22:39:21,943 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9cbf4ade36acb62156f78dbcf26aa02e:B 2024-11-10T22:39:21,943 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T22:39:21,944 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742256_1432 (size=31937) 2024-11-10T22:39:21,945 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37183 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T22:39:21,945 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1541): 9cbf4ade36acb62156f78dbcf26aa02e/C is initiating minor compaction (all files) 2024-11-10T22:39:21,945 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9cbf4ade36acb62156f78dbcf26aa02e/C in TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. 2024-11-10T22:39:21,945 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/C/7729a1aae8a84965b3ec78898a6f096e, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/C/e27d134d41d349e58087dff558a29b98, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/C/1f45420e2bae436b8f1affbf32b91a3d] into tmpdir=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp, totalSize=36.3 K 2024-11-10T22:39:21,945 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting 7729a1aae8a84965b3ec78898a6f096e, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731278357279 2024-11-10T22:39:21,946 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting e27d134d41d349e58087dff558a29b98, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1731278357365 2024-11-10T22:39:21,946 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting 1f45420e2bae436b8f1affbf32b91a3d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1731278359450 2024-11-10T22:39:21,953 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9cbf4ade36acb62156f78dbcf26aa02e#C#compaction#358 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T22:39:21,953 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/C/33e7bd7e1889400fb42e722acf302851 is 50, key is test_row_0/C:col10/1731278359462/Put/seqid=0 2024-11-10T22:39:21,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742257_1433 (size=12983) 2024-11-10T22:39:21,961 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/C/33e7bd7e1889400fb42e722acf302851 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/C/33e7bd7e1889400fb42e722acf302851 2024-11-10T22:39:21,966 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9cbf4ade36acb62156f78dbcf26aa02e/C of 9cbf4ade36acb62156f78dbcf26aa02e into 33e7bd7e1889400fb42e722acf302851(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T22:39:21,966 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9cbf4ade36acb62156f78dbcf26aa02e: 2024-11-10T22:39:21,966 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., storeName=9cbf4ade36acb62156f78dbcf26aa02e/C, priority=13, startTime=1731278361912; duration=0sec 2024-11-10T22:39:21,966 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T22:39:21,966 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9cbf4ade36acb62156f78dbcf26aa02e:C 2024-11-10T22:39:21,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(8855): Flush requested on 9cbf4ade36acb62156f78dbcf26aa02e 2024-11-10T22:39:21,980 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T22:39:21,980 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9cbf4ade36acb62156f78dbcf26aa02e 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-10T22:39:21,987 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110a92068d3ef464331b2072f3659e913d1_9cbf4ade36acb62156f78dbcf26aa02e is 50, key is test_row_0/A:col10/1731278360754/Put/seqid=0 2024-11-10T22:39:21,988 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-10T22:39:21,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. 2024-11-10T22:39:21,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. as already flushing 2024-11-10T22:39:21,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. 2024-11-10T22:39:21,988 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 9cbf4ade36acb62156f78dbcf26aa02e, NAME => 'TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:21,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 9cbf4ade36acb62156f78dbcf26aa02e, NAME => 'TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:21,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9cbf4ade36acb62156f78dbcf26aa02e, NAME => 'TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9cbf4ade36acb62156f78dbcf26aa02e, NAME => 'TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:21,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742258_1434 (size=14994) 2024-11-10T22:39:22,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-10T22:39:22,056 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:22,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60796 deadline: 1731278422050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:22,056 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:22,056 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:22,056 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:22,142 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-10T22:39:22,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. 2024-11-10T22:39:22,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. as already flushing 2024-11-10T22:39:22,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. 2024-11-10T22:39:22,143 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 9cbf4ade36acb62156f78dbcf26aa02e, NAME => 'TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:22,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 9cbf4ade36acb62156f78dbcf26aa02e, NAME => 'TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:22,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9cbf4ade36acb62156f78dbcf26aa02e, NAME => 'TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9cbf4ade36acb62156f78dbcf26aa02e, NAME => 'TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:22,168 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:22,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60796 deadline: 1731278422162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:22,169 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:22,169 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:22,169 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:22,295 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-10T22:39:22,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. 2024-11-10T22:39:22,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. as already flushing 2024-11-10T22:39:22,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. 2024-11-10T22:39:22,296 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 9cbf4ade36acb62156f78dbcf26aa02e, NAME => 'TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:22,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 9cbf4ade36acb62156f78dbcf26aa02e, NAME => 'TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:22,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9cbf4ade36acb62156f78dbcf26aa02e, NAME => 'TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9cbf4ade36acb62156f78dbcf26aa02e, NAME => 'TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:22,348 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/A/0bb00ed892114513b0b8680f90fcdd00 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/A/0bb00ed892114513b0b8680f90fcdd00 2024-11-10T22:39:22,353 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9cbf4ade36acb62156f78dbcf26aa02e/A of 9cbf4ade36acb62156f78dbcf26aa02e into 0bb00ed892114513b0b8680f90fcdd00(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T22:39:22,353 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9cbf4ade36acb62156f78dbcf26aa02e: 2024-11-10T22:39:22,353 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., storeName=9cbf4ade36acb62156f78dbcf26aa02e/A, priority=13, startTime=1731278361911; duration=0sec 2024-11-10T22:39:22,353 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T22:39:22,353 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9cbf4ade36acb62156f78dbcf26aa02e:A 2024-11-10T22:39:22,378 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:22,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60796 deadline: 1731278422372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:22,378 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:22,378 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:22,378 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:22,394 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:22,398 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110a92068d3ef464331b2072f3659e913d1_9cbf4ade36acb62156f78dbcf26aa02e to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110a92068d3ef464331b2072f3659e913d1_9cbf4ade36acb62156f78dbcf26aa02e 2024-11-10T22:39:22,399 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/A/da5d82e0d6d04de7abfa6c758fcdb51e, store: [table=TestAcidGuarantees family=A region=9cbf4ade36acb62156f78dbcf26aa02e] 2024-11-10T22:39:22,399 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/A/da5d82e0d6d04de7abfa6c758fcdb51e is 175, key is test_row_0/A:col10/1731278360754/Put/seqid=0 2024-11-10T22:39:22,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742259_1435 (size=39949) 2024-11-10T22:39:22,448 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-10T22:39:22,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. 2024-11-10T22:39:22,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. as already flushing 2024-11-10T22:39:22,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. 2024-11-10T22:39:22,449 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 9cbf4ade36acb62156f78dbcf26aa02e, NAME => 'TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:22,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 9cbf4ade36acb62156f78dbcf26aa02e, NAME => 'TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:22,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9cbf4ade36acb62156f78dbcf26aa02e, NAME => 'TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9cbf4ade36acb62156f78dbcf26aa02e, NAME => 'TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:22,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-10T22:39:22,602 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-10T22:39:22,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. 2024-11-10T22:39:22,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. as already flushing 2024-11-10T22:39:22,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. 2024-11-10T22:39:22,603 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 9cbf4ade36acb62156f78dbcf26aa02e, NAME => 'TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:22,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 9cbf4ade36acb62156f78dbcf26aa02e, NAME => 'TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:22,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9cbf4ade36acb62156f78dbcf26aa02e, NAME => 'TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9cbf4ade36acb62156f78dbcf26aa02e, NAME => 'TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:22,692 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:22,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60796 deadline: 1731278422682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:22,692 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:22,692 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:22,692 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:22,755 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-10T22:39:22,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. 2024-11-10T22:39:22,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. as already flushing 2024-11-10T22:39:22,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. 2024-11-10T22:39:22,756 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 9cbf4ade36acb62156f78dbcf26aa02e, NAME => 'TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:22,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 9cbf4ade36acb62156f78dbcf26aa02e, NAME => 'TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:22,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9cbf4ade36acb62156f78dbcf26aa02e, NAME => 'TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9cbf4ade36acb62156f78dbcf26aa02e, NAME => 'TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:22,803 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=298, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/A/da5d82e0d6d04de7abfa6c758fcdb51e 2024-11-10T22:39:22,810 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/B/a7cb3b016900405c99027701cea22ebf is 50, key is test_row_0/B:col10/1731278360754/Put/seqid=0 2024-11-10T22:39:22,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742260_1436 (size=12301) 2024-11-10T22:39:22,908 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-10T22:39:22,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. 2024-11-10T22:39:22,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. as already flushing 2024-11-10T22:39:22,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. 2024-11-10T22:39:22,909 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 9cbf4ade36acb62156f78dbcf26aa02e, NAME => 'TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:22,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 9cbf4ade36acb62156f78dbcf26aa02e, NAME => 'TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:22,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9cbf4ade36acb62156f78dbcf26aa02e, NAME => 'TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9cbf4ade36acb62156f78dbcf26aa02e, NAME => 'TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:23,062 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-10T22:39:23,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. 2024-11-10T22:39:23,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. as already flushing 2024-11-10T22:39:23,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. 2024-11-10T22:39:23,063 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 9cbf4ade36acb62156f78dbcf26aa02e, NAME => 'TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:23,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 9cbf4ade36acb62156f78dbcf26aa02e, NAME => 'TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:23,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9cbf4ade36acb62156f78dbcf26aa02e, NAME => 'TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9cbf4ade36acb62156f78dbcf26aa02e, NAME => 'TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:23,207 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:23,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60796 deadline: 1731278423202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:23,207 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:23,207 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:23,207 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:39:23,216 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-10T22:39:23,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. 2024-11-10T22:39:23,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. as already flushing 2024-11-10T22:39:23,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. 2024-11-10T22:39:23,216 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 9cbf4ade36acb62156f78dbcf26aa02e, NAME => 'TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:23,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 9cbf4ade36acb62156f78dbcf26aa02e, NAME => 'TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:23,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9cbf4ade36acb62156f78dbcf26aa02e, NAME => 'TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9cbf4ade36acb62156f78dbcf26aa02e, NAME => 'TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:23,218 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=298 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/B/a7cb3b016900405c99027701cea22ebf 2024-11-10T22:39:23,225 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/C/94e5cff1dfc44c7d8b0e48f6b90220cf is 50, key is test_row_0/C:col10/1731278360754/Put/seqid=0 2024-11-10T22:39:23,228 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742261_1437 (size=12301) 2024-11-10T22:39:23,230 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=298 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/C/94e5cff1dfc44c7d8b0e48f6b90220cf 2024-11-10T22:39:23,233 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/A/da5d82e0d6d04de7abfa6c758fcdb51e as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/A/da5d82e0d6d04de7abfa6c758fcdb51e 2024-11-10T22:39:23,236 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/A/da5d82e0d6d04de7abfa6c758fcdb51e, entries=200, sequenceid=298, filesize=39.0 K 2024-11-10T22:39:23,237 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/B/a7cb3b016900405c99027701cea22ebf as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/B/a7cb3b016900405c99027701cea22ebf 2024-11-10T22:39:23,240 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/B/a7cb3b016900405c99027701cea22ebf, entries=150, sequenceid=298, filesize=12.0 K 2024-11-10T22:39:23,241 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/C/94e5cff1dfc44c7d8b0e48f6b90220cf as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/C/94e5cff1dfc44c7d8b0e48f6b90220cf 2024-11-10T22:39:23,244 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/C/94e5cff1dfc44c7d8b0e48f6b90220cf, entries=150, sequenceid=298, filesize=12.0 K 2024-11-10T22:39:23,245 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 9cbf4ade36acb62156f78dbcf26aa02e in 1265ms, sequenceid=298, compaction requested=false 2024-11-10T22:39:23,245 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9cbf4ade36acb62156f78dbcf26aa02e: 2024-11-10T22:39:23,274 INFO [Thread-1597 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1597. 2024-11-10T22:39:23,274 DEBUG [Thread-1597 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T22:39:23,275 DEBUG [Thread-1597 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:39:23,275 DEBUG [Thread-1597 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:39:23,275 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T22:39:23,275 INFO [Thread-1601 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1601. 2024-11-10T22:39:23,275 DEBUG [Thread-1601 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T22:39:23,275 DEBUG [Thread-1601 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:39:23,275 DEBUG [Thread-1601 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:39:23,275 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T22:39:23,276 INFO [Thread-1603 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1603. 2024-11-10T22:39:23,276 DEBUG [Thread-1603 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T22:39:23,276 DEBUG [Thread-1603 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:39:23,277 DEBUG [Thread-1603 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:39:23,277 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T22:39:23,281 INFO [Thread-1599 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1599. 2024-11-10T22:39:23,281 DEBUG [Thread-1599 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T22:39:23,281 DEBUG [Thread-1599 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:39:23,281 DEBUG [Thread-1599 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:39:23,281 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T22:39:23,281 INFO [Thread-1595 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1595. 2024-11-10T22:39:23,281 DEBUG [Thread-1595 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T22:39:23,281 DEBUG [Thread-1595 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:39:23,281 DEBUG [Thread-1595 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:39:23,281 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T22:39:23,369 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-10T22:39:23,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. 2024-11-10T22:39:23,369 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2902): Flushing 9cbf4ade36acb62156f78dbcf26aa02e 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-10T22:39:23,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411104599bde358c04e959fc0bc1572848e86_9cbf4ade36acb62156f78dbcf26aa02e is 50, key is test_row_0/A:col10/1731278362033/Put/seqid=0 2024-11-10T22:39:23,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742262_1438 (size=12454) 2024-11-10T22:39:23,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-10T22:39:23,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:23,780 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411104599bde358c04e959fc0bc1572848e86_9cbf4ade36acb62156f78dbcf26aa02e to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411104599bde358c04e959fc0bc1572848e86_9cbf4ade36acb62156f78dbcf26aa02e 2024-11-10T22:39:23,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/A/939979cd8ecf4107a1bb05c60697e209, store: [table=TestAcidGuarantees family=A region=9cbf4ade36acb62156f78dbcf26aa02e] 2024-11-10T22:39:23,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/A/939979cd8ecf4107a1bb05c60697e209 is 175, key is test_row_0/A:col10/1731278362033/Put/seqid=0 2024-11-10T22:39:23,784 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742263_1439 (size=31255) 2024-11-10T22:39:24,185 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=312, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/A/939979cd8ecf4107a1bb05c60697e209 2024-11-10T22:39:24,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/B/32f5716d8a9348629ab796f272d1dc95 is 50, key is test_row_0/B:col10/1731278362033/Put/seqid=0 2024-11-10T22:39:24,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742264_1440 (size=12301) 2024-11-10T22:39:24,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(8855): Flush requested on 9cbf4ade36acb62156f78dbcf26aa02e 2024-11-10T22:39:24,223 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. as already flushing 2024-11-10T22:39:24,224 INFO [Thread-1584 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1584. 2024-11-10T22:39:24,224 DEBUG [Thread-1584 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T22:39:24,224 DEBUG [Thread-1584 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:39:24,224 DEBUG [Thread-1584 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:39:24,224 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T22:39:24,594 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=312 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/B/32f5716d8a9348629ab796f272d1dc95 2024-11-10T22:39:24,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/C/9e4aaf49ef58408faf033c1c2fc4cfb4 is 50, key is test_row_0/C:col10/1731278362033/Put/seqid=0 2024-11-10T22:39:24,603 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742265_1441 (size=12301) 2024-11-10T22:39:25,004 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=312 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/C/9e4aaf49ef58408faf033c1c2fc4cfb4 2024-11-10T22:39:25,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/A/939979cd8ecf4107a1bb05c60697e209 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/A/939979cd8ecf4107a1bb05c60697e209 2024-11-10T22:39:25,010 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/A/939979cd8ecf4107a1bb05c60697e209, entries=150, sequenceid=312, filesize=30.5 K 2024-11-10T22:39:25,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/B/32f5716d8a9348629ab796f272d1dc95 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/B/32f5716d8a9348629ab796f272d1dc95 2024-11-10T22:39:25,014 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/B/32f5716d8a9348629ab796f272d1dc95, entries=150, sequenceid=312, filesize=12.0 K 2024-11-10T22:39:25,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/C/9e4aaf49ef58408faf033c1c2fc4cfb4 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/C/9e4aaf49ef58408faf033c1c2fc4cfb4 2024-11-10T22:39:25,017 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/C/9e4aaf49ef58408faf033c1c2fc4cfb4, entries=150, sequenceid=312, filesize=12.0 K 2024-11-10T22:39:25,018 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=6.71 KB/6870 for 9cbf4ade36acb62156f78dbcf26aa02e in 1649ms, sequenceid=312, compaction requested=true 2024-11-10T22:39:25,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2603): Flush status journal for 9cbf4ade36acb62156f78dbcf26aa02e: 2024-11-10T22:39:25,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. 2024-11-10T22:39:25,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=122 2024-11-10T22:39:25,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43351 {}] master.HMaster(4169): Remote procedure done, pid=122 2024-11-10T22:39:25,020 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-11-10T22:39:25,020 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.6430 sec 2024-11-10T22:39:25,022 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees in 3.6480 sec 2024-11-10T22:39:25,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-10T22:39:25,533 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T22:39:25,623 INFO [Thread-1588 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1588. 2024-11-10T22:39:25,624 DEBUG [Thread-1588 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T22:39:25,624 DEBUG [Thread-1588 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:39:25,624 DEBUG [Thread-1588 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:39:25,624 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T22:39:25,634 INFO [Thread-1592 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1592. 2024-11-10T22:39:25,634 DEBUG [Thread-1592 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T22:39:25,634 DEBUG [Thread-1592 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:39:25,634 DEBUG [Thread-1592 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:39:25,634 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T22:39:25,674 INFO [Thread-1590 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1590. 2024-11-10T22:39:25,674 DEBUG [Thread-1590 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T22:39:25,674 DEBUG [Thread-1590 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:39:25,674 DEBUG [Thread-1590 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:39:25,674 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T22:39:27,603 INFO [Thread-1586 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1586. 2024-11-10T22:39:27,603 DEBUG [Thread-1586 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T22:39:27,603 DEBUG [Thread-1586 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:39:27,603 DEBUG [Thread-1586 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:39:27,603 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T22:39:27,604 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-10T22:39:27,604 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 80 2024-11-10T22:39:27,604 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 21 2024-11-10T22:39:27,604 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 41 2024-11-10T22:39:27,604 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 56 2024-11-10T22:39:27,604 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 44 2024-11-10T22:39:27,604 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-10T22:39:27,604 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-10T22:39:27,604 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1127 2024-11-10T22:39:27,604 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3380 rows 2024-11-10T22:39:27,604 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1126 2024-11-10T22:39:27,604 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3374 rows 2024-11-10T22:39:27,604 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1124 2024-11-10T22:39:27,604 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3371 rows 2024-11-10T22:39:27,604 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1129 2024-11-10T22:39:27,604 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3383 rows 2024-11-10T22:39:27,604 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1126 2024-11-10T22:39:27,604 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3376 rows 2024-11-10T22:39:27,604 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-10T22:39:27,604 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T22:39:27,604 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:39:27,604 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:39:27,604 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T22:39:27,605 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43351 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-10T22:39:27,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43351 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-10T22:39:27,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-10T22:39:27,608 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731278367607"}]},"ts":"1731278367607"} 2024-11-10T22:39:27,609 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-10T22:39:27,609 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-10T22:39:27,609 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-10T22:39:27,611 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=125, ppid=124, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9cbf4ade36acb62156f78dbcf26aa02e, UNASSIGN}] 2024-11-10T22:39:27,611 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=125, ppid=124, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9cbf4ade36acb62156f78dbcf26aa02e, UNASSIGN 2024-11-10T22:39:27,612 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=125 updating hbase:meta row=9cbf4ade36acb62156f78dbcf26aa02e, regionState=CLOSING, regionLocation=849d0d71310f,43753,1731278250663 2024-11-10T22:39:27,613 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=125, ppid=124, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=9cbf4ade36acb62156f78dbcf26aa02e, UNASSIGN because future has completed 2024-11-10T22:39:27,613 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-10T22:39:27,613 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; CloseRegionProcedure 9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663}] 2024-11-10T22:39:27,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-10T22:39:27,765 INFO [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] handler.UnassignRegionHandler(122): Close 9cbf4ade36acb62156f78dbcf26aa02e 2024-11-10T22:39:27,765 DEBUG [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-10T22:39:27,765 DEBUG [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.HRegion(1722): Closing 9cbf4ade36acb62156f78dbcf26aa02e, disabling compactions & flushes 2024-11-10T22:39:27,766 INFO [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. 2024-11-10T22:39:27,766 DEBUG [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. 2024-11-10T22:39:27,766 DEBUG [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. after waiting 0 ms 2024-11-10T22:39:27,766 DEBUG [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. 2024-11-10T22:39:27,766 INFO [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.HRegion(2902): Flushing 9cbf4ade36acb62156f78dbcf26aa02e 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-10T22:39:27,771 DEBUG [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411109a0e19eb63334e7ab54674e9abb10e7e_9cbf4ade36acb62156f78dbcf26aa02e is 50, key is test_row_0/A:col10/1731278365673/Put/seqid=0 2024-11-10T22:39:27,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742266_1442 (size=9914) 2024-11-10T22:39:27,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-10T22:39:28,175 DEBUG [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:28,179 INFO [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411109a0e19eb63334e7ab54674e9abb10e7e_9cbf4ade36acb62156f78dbcf26aa02e to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411109a0e19eb63334e7ab54674e9abb10e7e_9cbf4ade36acb62156f78dbcf26aa02e 2024-11-10T22:39:28,180 DEBUG [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/A/c8b6f863484748428b7900d7f8a461b9, store: [table=TestAcidGuarantees family=A region=9cbf4ade36acb62156f78dbcf26aa02e] 2024-11-10T22:39:28,180 DEBUG [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/A/c8b6f863484748428b7900d7f8a461b9 is 175, key is test_row_0/A:col10/1731278365673/Put/seqid=0 2024-11-10T22:39:28,183 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742267_1443 (size=22561) 2024-11-10T22:39:28,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-10T22:39:28,584 INFO [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=320, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/A/c8b6f863484748428b7900d7f8a461b9 2024-11-10T22:39:28,589 DEBUG [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/B/eb18cfa83cdb4c8487fbac5326d82e6f is 50, key is test_row_0/B:col10/1731278365673/Put/seqid=0 2024-11-10T22:39:28,592 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742268_1444 (size=9857) 2024-11-10T22:39:28,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-10T22:39:28,993 INFO [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=320 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/B/eb18cfa83cdb4c8487fbac5326d82e6f 2024-11-10T22:39:28,998 DEBUG [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/C/34f07cbe8f21494a9c3e07ea27605c17 is 50, key is test_row_0/C:col10/1731278365673/Put/seqid=0 2024-11-10T22:39:29,001 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742269_1445 (size=9857) 2024-11-10T22:39:29,063 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-10T22:39:29,402 INFO [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=320 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/C/34f07cbe8f21494a9c3e07ea27605c17 2024-11-10T22:39:29,405 DEBUG [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/A/c8b6f863484748428b7900d7f8a461b9 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/A/c8b6f863484748428b7900d7f8a461b9 2024-11-10T22:39:29,408 INFO [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/A/c8b6f863484748428b7900d7f8a461b9, entries=100, sequenceid=320, filesize=22.0 K 2024-11-10T22:39:29,409 DEBUG [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/B/eb18cfa83cdb4c8487fbac5326d82e6f as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/B/eb18cfa83cdb4c8487fbac5326d82e6f 2024-11-10T22:39:29,412 INFO [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/B/eb18cfa83cdb4c8487fbac5326d82e6f, entries=100, sequenceid=320, filesize=9.6 K 2024-11-10T22:39:29,412 DEBUG [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/.tmp/C/34f07cbe8f21494a9c3e07ea27605c17 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/C/34f07cbe8f21494a9c3e07ea27605c17 2024-11-10T22:39:29,415 INFO [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/C/34f07cbe8f21494a9c3e07ea27605c17, entries=100, sequenceid=320, filesize=9.6 K 2024-11-10T22:39:29,416 INFO [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 9cbf4ade36acb62156f78dbcf26aa02e in 1650ms, sequenceid=320, compaction requested=true 2024-11-10T22:39:29,417 DEBUG [StoreCloser-TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/A/ebc3917ea86d4b548ba7ba1f496355cc, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/A/36bc770484ee4d2983b226fcb614b6f4, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/A/a91e85b1f07c43f8b0be9bdaaf687e0f, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/A/13260c0ea97e473eacbe152329b99485, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/A/7b36acb88163429e98464f016a6453d9, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/A/222d356529fc407694ed7aa935f9647f, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/A/74e097657c674611b18587e088c32f3f, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/A/0e4b2856efb54e80a9242a51e30a581e, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/A/1e10cf5907b64dc18561cbe9df3d4acd, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/A/287461cb233043238245cf320e44ed9a, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/A/e7645db71df9468687c0c48eaf5098f4, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/A/f614b3b6b82041b48437a834f7cd6ae1, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/A/209ae414f2154b30b68e6a5d17bdf3f3, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/A/0166b5db2af9439a8338262bfa2a289d, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/A/93a34056eca94ff2b291854df15f28bd, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/A/6c009ce537514052b852fc5afd829096, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/A/f5714406dbc0475bae0919cbc261bbae, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/A/fa55a8ad318e4811accba285c0530ad1, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/A/5973fccc77d7405f8a0ee8dcaeb151e5, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/A/52b01f937b514165b9bad76d49ffc2fe] to archive 2024-11-10T22:39:29,417 DEBUG [StoreCloser-TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T22:39:29,418 DEBUG [StoreCloser-TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/A/ebc3917ea86d4b548ba7ba1f496355cc to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/A/ebc3917ea86d4b548ba7ba1f496355cc 2024-11-10T22:39:29,419 DEBUG [StoreCloser-TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/A/36bc770484ee4d2983b226fcb614b6f4 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/A/36bc770484ee4d2983b226fcb614b6f4 2024-11-10T22:39:29,420 DEBUG [StoreCloser-TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/A/a91e85b1f07c43f8b0be9bdaaf687e0f to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/A/a91e85b1f07c43f8b0be9bdaaf687e0f 2024-11-10T22:39:29,421 DEBUG [StoreCloser-TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/A/13260c0ea97e473eacbe152329b99485 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/A/13260c0ea97e473eacbe152329b99485 2024-11-10T22:39:29,422 DEBUG [StoreCloser-TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/A/7b36acb88163429e98464f016a6453d9 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/A/7b36acb88163429e98464f016a6453d9 2024-11-10T22:39:29,423 DEBUG [StoreCloser-TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/A/222d356529fc407694ed7aa935f9647f to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/A/222d356529fc407694ed7aa935f9647f 2024-11-10T22:39:29,423 DEBUG [StoreCloser-TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/A/74e097657c674611b18587e088c32f3f to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/A/74e097657c674611b18587e088c32f3f 2024-11-10T22:39:29,424 DEBUG [StoreCloser-TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/A/0e4b2856efb54e80a9242a51e30a581e to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/A/0e4b2856efb54e80a9242a51e30a581e 2024-11-10T22:39:29,425 DEBUG [StoreCloser-TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/A/1e10cf5907b64dc18561cbe9df3d4acd to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/A/1e10cf5907b64dc18561cbe9df3d4acd 2024-11-10T22:39:29,426 DEBUG [StoreCloser-TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/A/287461cb233043238245cf320e44ed9a to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/A/287461cb233043238245cf320e44ed9a 2024-11-10T22:39:29,426 DEBUG [StoreCloser-TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/A/e7645db71df9468687c0c48eaf5098f4 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/A/e7645db71df9468687c0c48eaf5098f4 2024-11-10T22:39:29,427 DEBUG [StoreCloser-TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/A/f614b3b6b82041b48437a834f7cd6ae1 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/A/f614b3b6b82041b48437a834f7cd6ae1 2024-11-10T22:39:29,428 DEBUG [StoreCloser-TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/A/209ae414f2154b30b68e6a5d17bdf3f3 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/A/209ae414f2154b30b68e6a5d17bdf3f3 2024-11-10T22:39:29,429 DEBUG [StoreCloser-TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/A/0166b5db2af9439a8338262bfa2a289d to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/A/0166b5db2af9439a8338262bfa2a289d 2024-11-10T22:39:29,430 DEBUG [StoreCloser-TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/A/93a34056eca94ff2b291854df15f28bd to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/A/93a34056eca94ff2b291854df15f28bd 2024-11-10T22:39:29,431 DEBUG [StoreCloser-TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/A/6c009ce537514052b852fc5afd829096 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/A/6c009ce537514052b852fc5afd829096 2024-11-10T22:39:29,432 DEBUG [StoreCloser-TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/A/f5714406dbc0475bae0919cbc261bbae to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/A/f5714406dbc0475bae0919cbc261bbae 2024-11-10T22:39:29,432 DEBUG [StoreCloser-TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/A/fa55a8ad318e4811accba285c0530ad1 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/A/fa55a8ad318e4811accba285c0530ad1 2024-11-10T22:39:29,433 DEBUG [StoreCloser-TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/A/5973fccc77d7405f8a0ee8dcaeb151e5 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/A/5973fccc77d7405f8a0ee8dcaeb151e5 2024-11-10T22:39:29,434 DEBUG [StoreCloser-TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/A/52b01f937b514165b9bad76d49ffc2fe to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/A/52b01f937b514165b9bad76d49ffc2fe 2024-11-10T22:39:29,435 DEBUG [StoreCloser-TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/B/df23d130d3de4aca8cd73a2c83ca92ad, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/B/584f11543f0449d281f60660321bd01f, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/B/1062828ffdb847b985e82ee173ff9f0a, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/B/3a4d01b97717458ba99b93e673b8d35c, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/B/6b554e8a86c346bea527af6f9cf4ad2a, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/B/b0da1e91f68743f8b9efd1e9e3278212, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/B/1ec14e8206cf42a7b8eda24f4135f0d6, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/B/6bbe0f099df24b65b7f9c3b6854cf6fc, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/B/5d4c589dcd524568ab44fe191b1ea5c7, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/B/e746fc0f9b854c7c82cbbc64ebd27c25, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/B/506a13e71a3b4f6888a8fdea226547da, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/B/39e30713469a4135b6d9270c50f6d551, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/B/5f0bdfa23bdd400db0af179347044626, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/B/19e5de7d2a92462883bc0718c4683117, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/B/e7199917fcbd49ffa940849c2d999f31, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/B/6b5829f8c884488ab141cd2b6d9feaa9, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/B/b1e8b376d276484eb51e8f181ed1b1e0, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/B/de2c518c7d3b410baf03fd42bc6e624a, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/B/123f1b13f83a481cb022a97fab82bca0, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/B/c47556c842d945c7bf1add7de697f6b2] to archive 2024-11-10T22:39:29,436 DEBUG [StoreCloser-TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T22:39:29,437 DEBUG [StoreCloser-TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/B/df23d130d3de4aca8cd73a2c83ca92ad to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/B/df23d130d3de4aca8cd73a2c83ca92ad 2024-11-10T22:39:29,438 DEBUG [StoreCloser-TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/B/584f11543f0449d281f60660321bd01f to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/B/584f11543f0449d281f60660321bd01f 2024-11-10T22:39:29,438 DEBUG [StoreCloser-TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/B/1062828ffdb847b985e82ee173ff9f0a to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/B/1062828ffdb847b985e82ee173ff9f0a 2024-11-10T22:39:29,439 DEBUG [StoreCloser-TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/B/3a4d01b97717458ba99b93e673b8d35c to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/B/3a4d01b97717458ba99b93e673b8d35c 2024-11-10T22:39:29,440 DEBUG [StoreCloser-TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/B/6b554e8a86c346bea527af6f9cf4ad2a to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/B/6b554e8a86c346bea527af6f9cf4ad2a 2024-11-10T22:39:29,441 DEBUG [StoreCloser-TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/B/b0da1e91f68743f8b9efd1e9e3278212 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/B/b0da1e91f68743f8b9efd1e9e3278212 2024-11-10T22:39:29,442 DEBUG [StoreCloser-TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/B/1ec14e8206cf42a7b8eda24f4135f0d6 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/B/1ec14e8206cf42a7b8eda24f4135f0d6 2024-11-10T22:39:29,442 DEBUG [StoreCloser-TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/B/6bbe0f099df24b65b7f9c3b6854cf6fc to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/B/6bbe0f099df24b65b7f9c3b6854cf6fc 2024-11-10T22:39:29,443 DEBUG [StoreCloser-TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/B/5d4c589dcd524568ab44fe191b1ea5c7 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/B/5d4c589dcd524568ab44fe191b1ea5c7 2024-11-10T22:39:29,444 DEBUG [StoreCloser-TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/B/e746fc0f9b854c7c82cbbc64ebd27c25 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/B/e746fc0f9b854c7c82cbbc64ebd27c25 2024-11-10T22:39:29,445 DEBUG [StoreCloser-TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/B/506a13e71a3b4f6888a8fdea226547da to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/B/506a13e71a3b4f6888a8fdea226547da 2024-11-10T22:39:29,445 DEBUG [StoreCloser-TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/B/39e30713469a4135b6d9270c50f6d551 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/B/39e30713469a4135b6d9270c50f6d551 2024-11-10T22:39:29,446 DEBUG [StoreCloser-TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/B/5f0bdfa23bdd400db0af179347044626 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/B/5f0bdfa23bdd400db0af179347044626 2024-11-10T22:39:29,447 DEBUG [StoreCloser-TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/B/19e5de7d2a92462883bc0718c4683117 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/B/19e5de7d2a92462883bc0718c4683117 2024-11-10T22:39:29,448 DEBUG [StoreCloser-TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/B/e7199917fcbd49ffa940849c2d999f31 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/B/e7199917fcbd49ffa940849c2d999f31 2024-11-10T22:39:29,449 DEBUG [StoreCloser-TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/B/6b5829f8c884488ab141cd2b6d9feaa9 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/B/6b5829f8c884488ab141cd2b6d9feaa9 2024-11-10T22:39:29,449 DEBUG [StoreCloser-TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/B/b1e8b376d276484eb51e8f181ed1b1e0 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/B/b1e8b376d276484eb51e8f181ed1b1e0 2024-11-10T22:39:29,450 DEBUG [StoreCloser-TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/B/de2c518c7d3b410baf03fd42bc6e624a to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/B/de2c518c7d3b410baf03fd42bc6e624a 2024-11-10T22:39:29,451 DEBUG [StoreCloser-TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/B/123f1b13f83a481cb022a97fab82bca0 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/B/123f1b13f83a481cb022a97fab82bca0 2024-11-10T22:39:29,452 DEBUG [StoreCloser-TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/B/c47556c842d945c7bf1add7de697f6b2 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/B/c47556c842d945c7bf1add7de697f6b2 2024-11-10T22:39:29,453 DEBUG [StoreCloser-TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/C/0d5ae8554bed48978fd73b35e356403c, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/C/526678fa872a424c8f907e1d5e2cdaca, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/C/1c506490e59544f39155fb152e819c75, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/C/17beedd7f8cb47f099efb2567a384f82, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/C/56d35fc3349f48418d8fad098d295e23, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/C/4d8c0e8c655940e2b6333815e0d1fe77, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/C/36103108c11440fe889482b5f583f5d9, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/C/240a76bbd06340c0b1ca6075a1257216, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/C/c81ed21ac94244cb9c6e5a65c7a889d5, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/C/11e45ea475f942a8b5a7c68099e5acca, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/C/f20877c160a14e7fbc65d4e390093ecf, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/C/5089b88c48ed4bd7b7456b7dec105213, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/C/337d06a60afc48d3905000ac5fb2842a, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/C/f9985c2b84d64198b93e7c260d5cb38a, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/C/dc6e32d0e3644122b747c5519db302da, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/C/34ad85bfc0484339b92a354111e585f6, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/C/7729a1aae8a84965b3ec78898a6f096e, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/C/74e04cdc74d0484ab4857d96ff51726a, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/C/e27d134d41d349e58087dff558a29b98, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/C/1f45420e2bae436b8f1affbf32b91a3d] to archive 2024-11-10T22:39:29,454 DEBUG [StoreCloser-TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T22:39:29,455 DEBUG [StoreCloser-TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/C/0d5ae8554bed48978fd73b35e356403c to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/C/0d5ae8554bed48978fd73b35e356403c 2024-11-10T22:39:29,456 DEBUG [StoreCloser-TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/C/526678fa872a424c8f907e1d5e2cdaca to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/C/526678fa872a424c8f907e1d5e2cdaca 2024-11-10T22:39:29,457 DEBUG [StoreCloser-TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/C/1c506490e59544f39155fb152e819c75 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/C/1c506490e59544f39155fb152e819c75 2024-11-10T22:39:29,458 DEBUG [StoreCloser-TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/C/17beedd7f8cb47f099efb2567a384f82 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/C/17beedd7f8cb47f099efb2567a384f82 2024-11-10T22:39:29,459 DEBUG [StoreCloser-TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/C/56d35fc3349f48418d8fad098d295e23 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/C/56d35fc3349f48418d8fad098d295e23 2024-11-10T22:39:29,460 DEBUG [StoreCloser-TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/C/4d8c0e8c655940e2b6333815e0d1fe77 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/C/4d8c0e8c655940e2b6333815e0d1fe77 2024-11-10T22:39:29,461 DEBUG [StoreCloser-TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/C/36103108c11440fe889482b5f583f5d9 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/C/36103108c11440fe889482b5f583f5d9 2024-11-10T22:39:29,461 DEBUG [StoreCloser-TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/C/240a76bbd06340c0b1ca6075a1257216 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/C/240a76bbd06340c0b1ca6075a1257216 2024-11-10T22:39:29,462 DEBUG [StoreCloser-TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/C/c81ed21ac94244cb9c6e5a65c7a889d5 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/C/c81ed21ac94244cb9c6e5a65c7a889d5 2024-11-10T22:39:29,463 DEBUG [StoreCloser-TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/C/11e45ea475f942a8b5a7c68099e5acca to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/C/11e45ea475f942a8b5a7c68099e5acca 2024-11-10T22:39:29,464 DEBUG [StoreCloser-TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/C/f20877c160a14e7fbc65d4e390093ecf to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/C/f20877c160a14e7fbc65d4e390093ecf 2024-11-10T22:39:29,465 DEBUG [StoreCloser-TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/C/5089b88c48ed4bd7b7456b7dec105213 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/C/5089b88c48ed4bd7b7456b7dec105213 2024-11-10T22:39:29,466 DEBUG [StoreCloser-TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/C/337d06a60afc48d3905000ac5fb2842a to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/C/337d06a60afc48d3905000ac5fb2842a 2024-11-10T22:39:29,467 DEBUG [StoreCloser-TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/C/f9985c2b84d64198b93e7c260d5cb38a to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/C/f9985c2b84d64198b93e7c260d5cb38a 2024-11-10T22:39:29,468 DEBUG [StoreCloser-TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/C/dc6e32d0e3644122b747c5519db302da to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/C/dc6e32d0e3644122b747c5519db302da 2024-11-10T22:39:29,469 DEBUG [StoreCloser-TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/C/34ad85bfc0484339b92a354111e585f6 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/C/34ad85bfc0484339b92a354111e585f6 2024-11-10T22:39:29,469 DEBUG [StoreCloser-TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/C/7729a1aae8a84965b3ec78898a6f096e to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/C/7729a1aae8a84965b3ec78898a6f096e 2024-11-10T22:39:29,470 DEBUG [StoreCloser-TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/C/74e04cdc74d0484ab4857d96ff51726a to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/C/74e04cdc74d0484ab4857d96ff51726a 2024-11-10T22:39:29,471 DEBUG [StoreCloser-TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/C/e27d134d41d349e58087dff558a29b98 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/C/e27d134d41d349e58087dff558a29b98 2024-11-10T22:39:29,472 DEBUG [StoreCloser-TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/C/1f45420e2bae436b8f1affbf32b91a3d to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/C/1f45420e2bae436b8f1affbf32b91a3d 2024-11-10T22:39:29,476 DEBUG [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/recovered.edits/323.seqid, newMaxSeqId=323, maxSeqId=4 2024-11-10T22:39:29,476 INFO [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. 2024-11-10T22:39:29,476 DEBUG [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.HRegion(1676): Region close journal for 9cbf4ade36acb62156f78dbcf26aa02e: Waiting for close lock at 1731278367765Running coprocessor pre-close hooks at 1731278367765Disabling compacts and flushes for region at 1731278367765Disabling writes for close at 1731278367766 (+1 ms)Obtaining lock to block concurrent updates at 1731278367766Preparing flush snapshotting stores in 9cbf4ade36acb62156f78dbcf26aa02e at 1731278367766Finished memstore snapshotting TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1731278367766Flushing stores of TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e. at 1731278367766Flushing 9cbf4ade36acb62156f78dbcf26aa02e/A: creating writer at 1731278367767 (+1 ms)Flushing mob file 9cbf4ade36acb62156f78dbcf26aa02e/A: appending metadata at 1731278367771 (+4 ms)Flushing mob file 9cbf4ade36acb62156f78dbcf26aa02e/A: closing flushed file at 1731278367771Flushing 9cbf4ade36acb62156f78dbcf26aa02e/A: appending metadata at 1731278368180 (+409 ms)Flushing 9cbf4ade36acb62156f78dbcf26aa02e/A: closing flushed file at 1731278368180Flushing 9cbf4ade36acb62156f78dbcf26aa02e/B: creating writer at 1731278368587 (+407 ms)Flushing 9cbf4ade36acb62156f78dbcf26aa02e/B: appending metadata at 1731278368589 (+2 ms)Flushing 9cbf4ade36acb62156f78dbcf26aa02e/B: closing flushed file at 1731278368589Flushing 9cbf4ade36acb62156f78dbcf26aa02e/C: creating writer at 1731278368996 (+407 ms)Flushing 9cbf4ade36acb62156f78dbcf26aa02e/C: appending metadata at 1731278368998 (+2 ms)Flushing 9cbf4ade36acb62156f78dbcf26aa02e/C: closing flushed file at 1731278368998Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1c58b7e3: reopening flushed file at 1731278369405 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2973ad08: reopening flushed file at 1731278369408 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@664932d1: reopening flushed file at 1731278369412 (+4 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 9cbf4ade36acb62156f78dbcf26aa02e in 1650ms, sequenceid=320, compaction requested=true at 1731278369416 (+4 ms)Writing region close event to WAL at 1731278369473 (+57 ms)Running coprocessor post-close hooks at 1731278369476 (+3 ms)Closed at 1731278369476 2024-11-10T22:39:29,478 INFO [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] handler.UnassignRegionHandler(157): Closed 9cbf4ade36acb62156f78dbcf26aa02e 2024-11-10T22:39:29,478 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=125 updating hbase:meta row=9cbf4ade36acb62156f78dbcf26aa02e, regionState=CLOSED 2024-11-10T22:39:29,480 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=126, ppid=125, state=RUNNABLE, hasLock=false; CloseRegionProcedure 9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 because future has completed 2024-11-10T22:39:29,482 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-11-10T22:39:29,482 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; CloseRegionProcedure 9cbf4ade36acb62156f78dbcf26aa02e, server=849d0d71310f,43753,1731278250663 in 1.8670 sec 2024-11-10T22:39:29,483 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=125, resume processing ppid=124 2024-11-10T22:39:29,483 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, ppid=124, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9cbf4ade36acb62156f78dbcf26aa02e, UNASSIGN in 1.8710 sec 2024-11-10T22:39:29,485 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-11-10T22:39:29,485 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8750 sec 2024-11-10T22:39:29,486 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731278369486"}]},"ts":"1731278369486"} 2024-11-10T22:39:29,488 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-10T22:39:29,488 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-10T22:39:29,489 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8830 sec 2024-11-10T22:39:29,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-10T22:39:29,752 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-10T22:39:29,753 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43351 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-10T22:39:29,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43351 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T22:39:29,755 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=127, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T22:39:29,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-10T22:39:29,755 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=127, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T22:39:29,757 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e 2024-11-10T22:39:29,759 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/A, FileablePath, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/B, FileablePath, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/C, FileablePath, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/recovered.edits] 2024-11-10T22:39:29,761 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/A/0bb00ed892114513b0b8680f90fcdd00 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/A/0bb00ed892114513b0b8680f90fcdd00 2024-11-10T22:39:29,762 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/A/939979cd8ecf4107a1bb05c60697e209 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/A/939979cd8ecf4107a1bb05c60697e209 2024-11-10T22:39:29,763 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/A/c8b6f863484748428b7900d7f8a461b9 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/A/c8b6f863484748428b7900d7f8a461b9 2024-11-10T22:39:29,764 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/A/da5d82e0d6d04de7abfa6c758fcdb51e to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/A/da5d82e0d6d04de7abfa6c758fcdb51e 2024-11-10T22:39:29,765 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/B/32f5716d8a9348629ab796f272d1dc95 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/B/32f5716d8a9348629ab796f272d1dc95 2024-11-10T22:39:29,766 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/B/a7cb3b016900405c99027701cea22ebf to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/B/a7cb3b016900405c99027701cea22ebf 2024-11-10T22:39:29,767 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/B/eb18cfa83cdb4c8487fbac5326d82e6f to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/B/eb18cfa83cdb4c8487fbac5326d82e6f 2024-11-10T22:39:29,768 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/B/f18d368743504cdc960c18153b8e9319 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/B/f18d368743504cdc960c18153b8e9319 2024-11-10T22:39:29,769 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/C/33e7bd7e1889400fb42e722acf302851 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/C/33e7bd7e1889400fb42e722acf302851 2024-11-10T22:39:29,770 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/C/34f07cbe8f21494a9c3e07ea27605c17 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/C/34f07cbe8f21494a9c3e07ea27605c17 2024-11-10T22:39:29,771 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/C/94e5cff1dfc44c7d8b0e48f6b90220cf to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/C/94e5cff1dfc44c7d8b0e48f6b90220cf 2024-11-10T22:39:29,772 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/C/9e4aaf49ef58408faf033c1c2fc4cfb4 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/C/9e4aaf49ef58408faf033c1c2fc4cfb4 2024-11-10T22:39:29,774 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/recovered.edits/323.seqid to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e/recovered.edits/323.seqid 2024-11-10T22:39:29,774 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/9cbf4ade36acb62156f78dbcf26aa02e 2024-11-10T22:39:29,774 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-10T22:39:29,775 DEBUG [PEWorker-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-10T22:39:29,775 DEBUG [PEWorker-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-10T22:39:29,777 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411102d26124b9b6c4083a68a8f791bbbc17c_9cbf4ade36acb62156f78dbcf26aa02e to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411102d26124b9b6c4083a68a8f791bbbc17c_9cbf4ade36acb62156f78dbcf26aa02e 2024-11-10T22:39:29,778 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110309b11d1f0a14ecc9d278c0f81d9bfb7_9cbf4ade36acb62156f78dbcf26aa02e to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110309b11d1f0a14ecc9d278c0f81d9bfb7_9cbf4ade36acb62156f78dbcf26aa02e 2024-11-10T22:39:29,779 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411103c4f9d662c164b9b82635210262018b3_9cbf4ade36acb62156f78dbcf26aa02e to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411103c4f9d662c164b9b82635210262018b3_9cbf4ade36acb62156f78dbcf26aa02e 2024-11-10T22:39:29,780 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111043661318fd3a402dad846d06eb3218db_9cbf4ade36acb62156f78dbcf26aa02e to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111043661318fd3a402dad846d06eb3218db_9cbf4ade36acb62156f78dbcf26aa02e 2024-11-10T22:39:29,780 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411104599bde358c04e959fc0bc1572848e86_9cbf4ade36acb62156f78dbcf26aa02e to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411104599bde358c04e959fc0bc1572848e86_9cbf4ade36acb62156f78dbcf26aa02e 2024-11-10T22:39:29,781 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110556fff8b927441b8a35fc0eeb50b54f0_9cbf4ade36acb62156f78dbcf26aa02e to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110556fff8b927441b8a35fc0eeb50b54f0_9cbf4ade36acb62156f78dbcf26aa02e 2024-11-10T22:39:29,782 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110570b908fcb42426299268f00bc40ff4d_9cbf4ade36acb62156f78dbcf26aa02e to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110570b908fcb42426299268f00bc40ff4d_9cbf4ade36acb62156f78dbcf26aa02e 2024-11-10T22:39:29,783 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110686b162dee7f4319bdf17ecd4f19fa84_9cbf4ade36acb62156f78dbcf26aa02e to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110686b162dee7f4319bdf17ecd4f19fa84_9cbf4ade36acb62156f78dbcf26aa02e 2024-11-10T22:39:29,783 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411107fef7a87e32041ccb2c53748815339b0_9cbf4ade36acb62156f78dbcf26aa02e to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411107fef7a87e32041ccb2c53748815339b0_9cbf4ade36acb62156f78dbcf26aa02e 2024-11-10T22:39:29,784 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110814b3608ed0b49189b21b027b82e2b32_9cbf4ade36acb62156f78dbcf26aa02e to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110814b3608ed0b49189b21b027b82e2b32_9cbf4ade36acb62156f78dbcf26aa02e 2024-11-10T22:39:29,785 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111094e985c4a58f4ecbb131ab3f2f141502_9cbf4ade36acb62156f78dbcf26aa02e to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111094e985c4a58f4ecbb131ab3f2f141502_9cbf4ade36acb62156f78dbcf26aa02e 2024-11-10T22:39:29,786 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110963a9a157e5547a484cd89f2e5b2bf9f_9cbf4ade36acb62156f78dbcf26aa02e to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110963a9a157e5547a484cd89f2e5b2bf9f_9cbf4ade36acb62156f78dbcf26aa02e 2024-11-10T22:39:29,786 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411109a0e19eb63334e7ab54674e9abb10e7e_9cbf4ade36acb62156f78dbcf26aa02e to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411109a0e19eb63334e7ab54674e9abb10e7e_9cbf4ade36acb62156f78dbcf26aa02e 2024-11-10T22:39:29,787 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110a92068d3ef464331b2072f3659e913d1_9cbf4ade36acb62156f78dbcf26aa02e to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110a92068d3ef464331b2072f3659e913d1_9cbf4ade36acb62156f78dbcf26aa02e 2024-11-10T22:39:29,788 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110b68ff42e6f7c48aea5d2516c6fa52106_9cbf4ade36acb62156f78dbcf26aa02e to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110b68ff42e6f7c48aea5d2516c6fa52106_9cbf4ade36acb62156f78dbcf26aa02e 2024-11-10T22:39:29,789 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110c12631847995440dab163e4e36a27155_9cbf4ade36acb62156f78dbcf26aa02e to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110c12631847995440dab163e4e36a27155_9cbf4ade36acb62156f78dbcf26aa02e 2024-11-10T22:39:29,789 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110d5cf7dd339344002aa0850bce9d39e30_9cbf4ade36acb62156f78dbcf26aa02e to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110d5cf7dd339344002aa0850bce9d39e30_9cbf4ade36acb62156f78dbcf26aa02e 2024-11-10T22:39:29,790 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110efd1c39fbd2a448dae7eaca9b7efa630_9cbf4ade36acb62156f78dbcf26aa02e to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110efd1c39fbd2a448dae7eaca9b7efa630_9cbf4ade36acb62156f78dbcf26aa02e 2024-11-10T22:39:29,791 DEBUG [PEWorker-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-10T22:39:29,792 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=127, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T22:39:29,794 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-10T22:39:29,796 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-10T22:39:29,797 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=127, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T22:39:29,797 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-10T22:39:29,797 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731278369797"}]},"ts":"9223372036854775807"} 2024-11-10T22:39:29,798 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-10T22:39:29,798 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 9cbf4ade36acb62156f78dbcf26aa02e, NAME => 'TestAcidGuarantees,,1731278340162.9cbf4ade36acb62156f78dbcf26aa02e.', STARTKEY => '', ENDKEY => ''}] 2024-11-10T22:39:29,798 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-10T22:39:29,799 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731278369798"}]},"ts":"9223372036854775807"} 2024-11-10T22:39:29,800 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-10T22:39:29,800 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=127, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T22:39:29,801 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 47 msec 2024-11-10T22:39:29,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-10T22:39:29,862 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-10T22:39:29,863 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-10T22:39:29,871 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobScanAtomicity Thread=238 (was 240), OpenFileDescriptor=453 (was 451) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=284 (was 295), ProcessCount=11 (was 11), AvailableMemoryMB=6951 (was 6994) 2024-11-10T22:39:29,879 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testScanAtomicity Thread=238, OpenFileDescriptor=453, MaxFileDescriptor=1048576, SystemLoadAverage=284, ProcessCount=11, AvailableMemoryMB=6951 2024-11-10T22:39:29,880 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43351 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-10T22:39:29,881 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43351 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-10T22:39:29,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43351 {}] procedure2.ProcedureExecutor(1139): Stored pid=128, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-10T22:39:29,883 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=128, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-10T22:39:29,883 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:29,883 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43351 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 128 2024-11-10T22:39:29,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=128 2024-11-10T22:39:29,883 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=128, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-10T22:39:29,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742270_1446 (size=959) 2024-11-10T22:39:29,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=128 2024-11-10T22:39:30,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=128 2024-11-10T22:39:30,290 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => ccb0ca518fd65dd7264624eb4a78050f, NAME => 'TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39 2024-11-10T22:39:30,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742271_1447 (size=53) 2024-11-10T22:39:30,392 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-10T22:39:30,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=128 2024-11-10T22:39:30,695 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T22:39:30,695 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing ccb0ca518fd65dd7264624eb4a78050f, disabling compactions & flushes 2024-11-10T22:39:30,695 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f. 2024-11-10T22:39:30,695 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f. 2024-11-10T22:39:30,695 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f. after waiting 0 ms 2024-11-10T22:39:30,695 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f. 2024-11-10T22:39:30,695 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f. 2024-11-10T22:39:30,695 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for ccb0ca518fd65dd7264624eb4a78050f: Waiting for close lock at 1731278370695Disabling compacts and flushes for region at 1731278370695Disabling writes for close at 1731278370695Writing region close event to WAL at 1731278370695Closed at 1731278370695 2024-11-10T22:39:30,696 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=128, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-10T22:39:30,696 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731278370696"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731278370696"}]},"ts":"1731278370696"} 2024-11-10T22:39:30,698 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-10T22:39:30,698 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=128, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-10T22:39:30,699 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731278370699"}]},"ts":"1731278370699"} 2024-11-10T22:39:30,700 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-10T22:39:30,700 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=129, ppid=128, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ccb0ca518fd65dd7264624eb4a78050f, ASSIGN}] 2024-11-10T22:39:30,701 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=129, ppid=128, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ccb0ca518fd65dd7264624eb4a78050f, ASSIGN 2024-11-10T22:39:30,701 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=129, ppid=128, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ccb0ca518fd65dd7264624eb4a78050f, ASSIGN; state=OFFLINE, location=849d0d71310f,43753,1731278250663; forceNewPlan=false, retain=false 2024-11-10T22:39:30,852 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=129 updating hbase:meta row=ccb0ca518fd65dd7264624eb4a78050f, regionState=OPENING, regionLocation=849d0d71310f,43753,1731278250663 2024-11-10T22:39:30,853 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=129, ppid=128, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ccb0ca518fd65dd7264624eb4a78050f, ASSIGN because future has completed 2024-11-10T22:39:30,854 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; OpenRegionProcedure ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663}] 2024-11-10T22:39:31,008 INFO [RS_OPEN_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f. 2024-11-10T22:39:31,008 DEBUG [RS_OPEN_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] regionserver.HRegion(7752): Opening region: {ENCODED => ccb0ca518fd65dd7264624eb4a78050f, NAME => 'TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.', STARTKEY => '', ENDKEY => ''} 2024-11-10T22:39:31,009 DEBUG [RS_OPEN_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees ccb0ca518fd65dd7264624eb4a78050f 2024-11-10T22:39:31,009 DEBUG [RS_OPEN_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T22:39:31,009 DEBUG [RS_OPEN_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] regionserver.HRegion(7794): checking encryption for ccb0ca518fd65dd7264624eb4a78050f 2024-11-10T22:39:31,009 DEBUG [RS_OPEN_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] regionserver.HRegion(7797): checking classloading for ccb0ca518fd65dd7264624eb4a78050f 2024-11-10T22:39:31,010 INFO [StoreOpener-ccb0ca518fd65dd7264624eb4a78050f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region ccb0ca518fd65dd7264624eb4a78050f 2024-11-10T22:39:31,011 INFO [StoreOpener-ccb0ca518fd65dd7264624eb4a78050f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ccb0ca518fd65dd7264624eb4a78050f columnFamilyName A 2024-11-10T22:39:31,011 DEBUG [StoreOpener-ccb0ca518fd65dd7264624eb4a78050f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:31,011 INFO [StoreOpener-ccb0ca518fd65dd7264624eb4a78050f-1 {}] regionserver.HStore(327): Store=ccb0ca518fd65dd7264624eb4a78050f/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T22:39:31,011 INFO [StoreOpener-ccb0ca518fd65dd7264624eb4a78050f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region ccb0ca518fd65dd7264624eb4a78050f 2024-11-10T22:39:31,012 INFO [StoreOpener-ccb0ca518fd65dd7264624eb4a78050f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ccb0ca518fd65dd7264624eb4a78050f columnFamilyName B 2024-11-10T22:39:31,012 DEBUG [StoreOpener-ccb0ca518fd65dd7264624eb4a78050f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:31,013 INFO [StoreOpener-ccb0ca518fd65dd7264624eb4a78050f-1 {}] regionserver.HStore(327): Store=ccb0ca518fd65dd7264624eb4a78050f/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T22:39:31,013 INFO [StoreOpener-ccb0ca518fd65dd7264624eb4a78050f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region ccb0ca518fd65dd7264624eb4a78050f 2024-11-10T22:39:31,013 INFO [StoreOpener-ccb0ca518fd65dd7264624eb4a78050f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ccb0ca518fd65dd7264624eb4a78050f columnFamilyName C 2024-11-10T22:39:31,013 DEBUG [StoreOpener-ccb0ca518fd65dd7264624eb4a78050f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:31,014 INFO [StoreOpener-ccb0ca518fd65dd7264624eb4a78050f-1 {}] regionserver.HStore(327): Store=ccb0ca518fd65dd7264624eb4a78050f/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T22:39:31,014 DEBUG [RS_OPEN_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] regionserver.HRegion(1038): replaying wal for ccb0ca518fd65dd7264624eb4a78050f 2024-11-10T22:39:31,014 DEBUG [RS_OPEN_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f 2024-11-10T22:39:31,015 DEBUG [RS_OPEN_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f 2024-11-10T22:39:31,015 DEBUG [RS_OPEN_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] regionserver.HRegion(1048): stopping wal replay for ccb0ca518fd65dd7264624eb4a78050f 2024-11-10T22:39:31,015 DEBUG [RS_OPEN_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] regionserver.HRegion(1060): Cleaning up temporary data for ccb0ca518fd65dd7264624eb4a78050f 2024-11-10T22:39:31,016 DEBUG [RS_OPEN_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-10T22:39:31,016 DEBUG [RS_OPEN_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] regionserver.HRegion(1093): writing seq id for ccb0ca518fd65dd7264624eb4a78050f 2024-11-10T22:39:31,018 DEBUG [RS_OPEN_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-10T22:39:31,018 INFO [RS_OPEN_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] regionserver.HRegion(1114): Opened ccb0ca518fd65dd7264624eb4a78050f; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72769562, jitterRate=0.08435097336769104}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-10T22:39:31,018 DEBUG [RS_OPEN_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ccb0ca518fd65dd7264624eb4a78050f 2024-11-10T22:39:31,019 DEBUG [RS_OPEN_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] regionserver.HRegion(1006): Region open journal for ccb0ca518fd65dd7264624eb4a78050f: Running coprocessor pre-open hook at 1731278371009Writing region info on filesystem at 1731278371009Initializing all the Stores at 1731278371009Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731278371009Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731278371010 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731278371010Cleaning up temporary data from old regions at 1731278371015 (+5 ms)Running coprocessor post-open hooks at 1731278371018 (+3 ms)Region opened successfully at 1731278371019 (+1 ms) 2024-11-10T22:39:31,019 INFO [RS_OPEN_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., pid=130, masterSystemTime=1731278371005 2024-11-10T22:39:31,021 DEBUG [RS_OPEN_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f. 2024-11-10T22:39:31,021 INFO [RS_OPEN_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f. 2024-11-10T22:39:31,021 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=129 updating hbase:meta row=ccb0ca518fd65dd7264624eb4a78050f, regionState=OPEN, openSeqNum=2, regionLocation=849d0d71310f,43753,1731278250663 2024-11-10T22:39:31,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=128 2024-11-10T22:39:31,022 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=130, ppid=129, state=RUNNABLE, hasLock=false; OpenRegionProcedure ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 because future has completed 2024-11-10T22:39:31,024 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-11-10T22:39:31,024 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; OpenRegionProcedure ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 in 169 msec 2024-11-10T22:39:31,026 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=129, resume processing ppid=128 2024-11-10T22:39:31,026 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, ppid=128, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ccb0ca518fd65dd7264624eb4a78050f, ASSIGN in 325 msec 2024-11-10T22:39:31,026 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=128, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-10T22:39:31,027 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731278371026"}]},"ts":"1731278371026"} 2024-11-10T22:39:31,028 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-10T22:39:31,028 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=128, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-10T22:39:31,029 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1470 sec 2024-11-10T22:39:32,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=128 2024-11-10T22:39:32,032 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-10T22:39:32,033 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@204ac613, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:39:32,033 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849d0d71310f,43351,-1 for getting cluster id 2024-11-10T22:39:32,033 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T22:39:32,034 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59ccc382-681c-4202-a6b6-e337b1afc6c7' 2024-11-10T22:39:32,034 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T22:39:32,034 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59ccc382-681c-4202-a6b6-e337b1afc6c7" 2024-11-10T22:39:32,035 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f074040, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:39:32,035 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849d0d71310f,43351,-1] 2024-11-10T22:39:32,035 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T22:39:32,035 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:39:32,036 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45716, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T22:39:32,036 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c8699be, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:39:32,037 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T22:39:32,037 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849d0d71310f,43753,1731278250663, seqNum=-1] 2024-11-10T22:39:32,037 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T22:39:32,038 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42312, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T22:39:32,039 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-10T22:39:32,039 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is 849d0d71310f,43351,1731278249979 2024-11-10T22:39:32,039 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@3536ec4f 2024-11-10T22:39:32,039 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-10T22:39:32,040 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45724, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-10T22:39:32,041 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74bdbad8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:39:32,041 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849d0d71310f,43351,-1 for getting cluster id 2024-11-10T22:39:32,041 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T22:39:32,042 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59ccc382-681c-4202-a6b6-e337b1afc6c7' 2024-11-10T22:39:32,042 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T22:39:32,042 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59ccc382-681c-4202-a6b6-e337b1afc6c7" 2024-11-10T22:39:32,042 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4cc87dd7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:39:32,042 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849d0d71310f,43351,-1] 2024-11-10T22:39:32,042 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T22:39:32,042 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:39:32,043 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45748, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T22:39:32,043 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ab56195, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:39:32,044 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ac69b7f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:39:32,044 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849d0d71310f,43351,-1 for getting cluster id 2024-11-10T22:39:32,044 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T22:39:32,044 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59ccc382-681c-4202-a6b6-e337b1afc6c7' 2024-11-10T22:39:32,044 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T22:39:32,044 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59ccc382-681c-4202-a6b6-e337b1afc6c7" 2024-11-10T22:39:32,045 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@471bafda, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:39:32,045 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849d0d71310f,43351,-1] 2024-11-10T22:39:32,045 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T22:39:32,045 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:39:32,045 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45772, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T22:39:32,046 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e88df86, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:39:32,046 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@277a5c68, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:39:32,046 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849d0d71310f,43351,-1 for getting cluster id 2024-11-10T22:39:32,046 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T22:39:32,047 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59ccc382-681c-4202-a6b6-e337b1afc6c7' 2024-11-10T22:39:32,047 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T22:39:32,047 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59ccc382-681c-4202-a6b6-e337b1afc6c7" 2024-11-10T22:39:32,047 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c4c0f92, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:39:32,047 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849d0d71310f,43351,-1] 2024-11-10T22:39:32,047 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T22:39:32,047 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:39:32,048 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45776, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T22:39:32,048 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15a705e2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:39:32,049 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1df5cdc7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:39:32,049 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849d0d71310f,43351,-1 for getting cluster id 2024-11-10T22:39:32,049 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T22:39:32,049 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59ccc382-681c-4202-a6b6-e337b1afc6c7' 2024-11-10T22:39:32,049 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T22:39:32,049 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59ccc382-681c-4202-a6b6-e337b1afc6c7" 2024-11-10T22:39:32,049 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@cb653f0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:39:32,049 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849d0d71310f,43351,-1] 2024-11-10T22:39:32,049 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T22:39:32,050 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:39:32,050 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45788, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T22:39:32,050 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b51a9fc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:39:32,051 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22b9b902, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:39:32,051 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849d0d71310f,43351,-1 for getting cluster id 2024-11-10T22:39:32,051 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T22:39:32,051 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59ccc382-681c-4202-a6b6-e337b1afc6c7' 2024-11-10T22:39:32,052 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T22:39:32,052 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59ccc382-681c-4202-a6b6-e337b1afc6c7" 2024-11-10T22:39:32,052 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@17d50142, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:39:32,052 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849d0d71310f,43351,-1] 2024-11-10T22:39:32,052 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T22:39:32,052 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:39:32,052 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45798, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T22:39:32,053 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a45f028, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:39:32,053 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f59c6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:39:32,053 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849d0d71310f,43351,-1 for getting cluster id 2024-11-10T22:39:32,053 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T22:39:32,054 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59ccc382-681c-4202-a6b6-e337b1afc6c7' 2024-11-10T22:39:32,054 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T22:39:32,054 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59ccc382-681c-4202-a6b6-e337b1afc6c7" 2024-11-10T22:39:32,054 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@181b7332, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:39:32,054 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849d0d71310f,43351,-1] 2024-11-10T22:39:32,054 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T22:39:32,054 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:39:32,055 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45816, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T22:39:32,055 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a75833a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:39:32,056 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ae789b8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:39:32,056 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849d0d71310f,43351,-1 for getting cluster id 2024-11-10T22:39:32,056 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T22:39:32,057 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59ccc382-681c-4202-a6b6-e337b1afc6c7' 2024-11-10T22:39:32,057 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T22:39:32,057 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59ccc382-681c-4202-a6b6-e337b1afc6c7" 2024-11-10T22:39:32,057 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@365728c9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:39:32,057 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849d0d71310f,43351,-1] 2024-11-10T22:39:32,057 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T22:39:32,057 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:39:32,058 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45832, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T22:39:32,058 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@384825cd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:39:32,059 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b7b3b89, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:39:32,059 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849d0d71310f,43351,-1 for getting cluster id 2024-11-10T22:39:32,059 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T22:39:32,059 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59ccc382-681c-4202-a6b6-e337b1afc6c7' 2024-11-10T22:39:32,059 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T22:39:32,059 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59ccc382-681c-4202-a6b6-e337b1afc6c7" 2024-11-10T22:39:32,060 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f172e70, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:39:32,060 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849d0d71310f,43351,-1] 2024-11-10T22:39:32,060 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T22:39:32,060 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:39:32,060 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45846, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T22:39:32,061 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55e1ccb0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:39:32,061 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e04f49c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:39:32,061 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849d0d71310f,43351,-1 for getting cluster id 2024-11-10T22:39:32,061 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T22:39:32,062 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59ccc382-681c-4202-a6b6-e337b1afc6c7' 2024-11-10T22:39:32,062 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T22:39:32,062 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59ccc382-681c-4202-a6b6-e337b1afc6c7" 2024-11-10T22:39:32,062 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61433c19, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:39:32,062 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849d0d71310f,43351,-1] 2024-11-10T22:39:32,062 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T22:39:32,062 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:39:32,063 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45854, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T22:39:32,063 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@755e66f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:39:32,064 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32ca199c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:39:32,064 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849d0d71310f,43351,-1 for getting cluster id 2024-11-10T22:39:32,064 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T22:39:32,064 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59ccc382-681c-4202-a6b6-e337b1afc6c7' 2024-11-10T22:39:32,064 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T22:39:32,065 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59ccc382-681c-4202-a6b6-e337b1afc6c7" 2024-11-10T22:39:32,065 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@322b447d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:39:32,065 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849d0d71310f,43351,-1] 2024-11-10T22:39:32,065 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T22:39:32,065 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:39:32,065 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45864, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T22:39:32,066 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b6c2158, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:39:32,066 DEBUG [Thread-2017 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T22:39:32,066 DEBUG [Thread-2010 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T22:39:32,066 DEBUG [Thread-2021 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T22:39:32,067 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849d0d71310f,43753,1731278250663, seqNum=-1] 2024-11-10T22:39:32,067 DEBUG [Thread-2008 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T22:39:32,067 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849d0d71310f,43753,1731278250663, seqNum=-1] 2024-11-10T22:39:32,067 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T22:39:32,067 DEBUG [Thread-2006 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T22:39:32,067 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T22:39:32,067 DEBUG [Thread-2012 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T22:39:32,068 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42316, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T22:39:32,068 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849d0d71310f,43753,1731278250663, seqNum=-1] 2024-11-10T22:39:32,068 DEBUG [Thread-2014 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T22:39:32,068 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849d0d71310f,43753,1731278250663, seqNum=-1] 2024-11-10T22:39:32,068 DEBUG [Thread-2023 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T22:39:32,068 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T22:39:32,068 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849d0d71310f,43753,1731278250663, seqNum=-1] 2024-11-10T22:39:32,068 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T22:39:32,068 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849d0d71310f,43753,1731278250663, seqNum=-1] 2024-11-10T22:39:32,069 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T22:39:32,069 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T22:39:32,069 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849d0d71310f,43753,1731278250663, seqNum=-1] 2024-11-10T22:39:32,069 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T22:39:32,069 DEBUG [Thread-2019 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T22:39:32,069 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42322, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T22:39:32,070 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T22:39:32,070 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2] 2024-11-10T22:39:32,070 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42328, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T22:39:32,070 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849d0d71310f,43753,1731278250663, seqNum=-1] 2024-11-10T22:39:32,070 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42346, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T22:39:32,070 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42356, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T22:39:32,070 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T22:39:32,071 DEBUG [Thread-2025 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T22:39:32,071 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42378, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T22:39:32,071 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42340, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T22:39:32,071 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2] 2024-11-10T22:39:32,072 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2] 2024-11-10T22:39:32,072 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42362, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T22:39:32,072 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849d0d71310f,43753,1731278250663, seqNum=-1] 2024-11-10T22:39:32,072 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849d0d71310f,43753,1731278250663, seqNum=-1] 2024-11-10T22:39:32,072 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T22:39:32,072 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T22:39:32,073 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2] 2024-11-10T22:39:32,073 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42380, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T22:39:32,073 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42396, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T22:39:32,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees 2024-11-10T22:39:32,074 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2] 2024-11-10T22:39:32,074 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2] 2024-11-10T22:39:32,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-10T22:39:32,075 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2] 2024-11-10T22:39:32,075 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2] 2024-11-10T22:39:32,075 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T22:39:32,075 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2] 2024-11-10T22:39:32,076 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2] 2024-11-10T22:39:32,076 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T22:39:32,077 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T22:39:32,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(8855): Flush requested on ccb0ca518fd65dd7264624eb4a78050f 2024-11-10T22:39:32,082 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T22:39:32,082 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ccb0ca518fd65dd7264624eb4a78050f 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-10T22:39:32,103 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/A/279bda6940fd43d68f05566362ff9f78 is 50, key is test_row_0/A:col10/1731278372080/Put/seqid=0 2024-11-10T22:39:32,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742272_1448 (size=14341) 2024-11-10T22:39:32,129 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:32,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42340 deadline: 1731278432118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:32,130 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:32,130 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:32,130 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:32,131 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:32,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42328 deadline: 1731278432119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:32,131 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:32,131 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:32,131 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:32,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42346 deadline: 1731278432121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:32,131 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:32,131 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:32,131 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:32,131 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:32,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42322 deadline: 1731278432130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:32,131 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:32,132 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:32,132 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:32,132 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:32,140 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:32,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42356 deadline: 1731278432130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:32,141 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:32,141 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:32,141 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:32,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-10T22:39:32,229 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-10T22:39:32,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f. 2024-11-10T22:39:32,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f. as already flushing 2024-11-10T22:39:32,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f. 2024-11-10T22:39:32,229 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => ccb0ca518fd65dd7264624eb4a78050f, NAME => 'TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:32,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => ccb0ca518fd65dd7264624eb4a78050f, NAME => 'TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:32,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ccb0ca518fd65dd7264624eb4a78050f, NAME => 'TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ccb0ca518fd65dd7264624eb4a78050f, NAME => 'TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:32,239 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:32,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42340 deadline: 1731278432233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:32,239 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:32,239 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:32,239 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:32,250 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:32,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42328 deadline: 1731278432243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:32,250 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:32,250 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:32,251 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:32,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42346 deadline: 1731278432243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:32,251 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:32,251 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:32,251 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:32,251 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:32,251 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:32,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42322 deadline: 1731278432243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:32,251 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:32,251 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:32,251 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:32,251 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:32,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42356 deadline: 1731278432243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:32,251 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:32,251 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:32,251 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:32,382 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-10T22:39:32,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f. 2024-11-10T22:39:32,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f. as already flushing 2024-11-10T22:39:32,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f. 2024-11-10T22:39:32,382 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => ccb0ca518fd65dd7264624eb4a78050f, NAME => 'TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:32,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => ccb0ca518fd65dd7264624eb4a78050f, NAME => 'TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:32,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ccb0ca518fd65dd7264624eb4a78050f, NAME => 'TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ccb0ca518fd65dd7264624eb4a78050f, NAME => 'TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:32,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-10T22:39:32,452 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:32,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42340 deadline: 1731278432443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:32,452 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:32,452 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:32,452 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:32,459 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:32,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42328 deadline: 1731278432452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:32,459 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:32,459 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:32,459 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:32,471 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:32,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42346 deadline: 1731278432462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:32,471 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:32,471 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:32,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42322 deadline: 1731278432462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:32,471 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:32,471 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:32,471 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:32,471 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:32,471 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:32,471 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:32,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42356 deadline: 1731278432463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:32,471 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:32,471 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:32,472 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:32,509 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/A/279bda6940fd43d68f05566362ff9f78 2024-11-10T22:39:32,532 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/B/aa0700e4a9234c1b8380465d65e362c7 is 50, key is test_row_0/B:col10/1731278372080/Put/seqid=0 2024-11-10T22:39:32,535 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-10T22:39:32,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f. 2024-11-10T22:39:32,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f. as already flushing 2024-11-10T22:39:32,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f. 2024-11-10T22:39:32,535 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => ccb0ca518fd65dd7264624eb4a78050f, NAME => 'TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:32,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => ccb0ca518fd65dd7264624eb4a78050f, NAME => 'TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:32,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742273_1449 (size=12001) 2024-11-10T22:39:32,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ccb0ca518fd65dd7264624eb4a78050f, NAME => 'TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ccb0ca518fd65dd7264624eb4a78050f, NAME => 'TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:32,688 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-10T22:39:32,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f. 2024-11-10T22:39:32,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f. as already flushing 2024-11-10T22:39:32,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f. 2024-11-10T22:39:32,689 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => ccb0ca518fd65dd7264624eb4a78050f, NAME => 'TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:32,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => ccb0ca518fd65dd7264624eb4a78050f, NAME => 'TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:32,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ccb0ca518fd65dd7264624eb4a78050f, NAME => 'TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ccb0ca518fd65dd7264624eb4a78050f, NAME => 'TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:32,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-10T22:39:32,766 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:32,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42340 deadline: 1731278432762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:32,766 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:32,766 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:32,766 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:32,778 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:32,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42328 deadline: 1731278432772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:32,778 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:32,778 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:32,778 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:32,790 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:32,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42346 deadline: 1731278432782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:32,790 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:32,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42322 deadline: 1731278432782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:32,790 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:32,790 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:32,790 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:32,790 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:32,790 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:32,790 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:32,790 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:32,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42356 deadline: 1731278432783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:32,791 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:32,791 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:32,791 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:32,841 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-10T22:39:32,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f. 2024-11-10T22:39:32,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f. as already flushing 2024-11-10T22:39:32,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f. 2024-11-10T22:39:32,842 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => ccb0ca518fd65dd7264624eb4a78050f, NAME => 'TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:32,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => ccb0ca518fd65dd7264624eb4a78050f, NAME => 'TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:32,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ccb0ca518fd65dd7264624eb4a78050f, NAME => 'TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ccb0ca518fd65dd7264624eb4a78050f, NAME => 'TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:32,937 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/B/aa0700e4a9234c1b8380465d65e362c7 2024-11-10T22:39:32,956 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/C/87d429300f2d47e59f9ffcf4de4d6a61 is 50, key is test_row_0/C:col10/1731278372080/Put/seqid=0 2024-11-10T22:39:32,959 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742274_1450 (size=12001) 2024-11-10T22:39:32,994 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-10T22:39:32,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f. 2024-11-10T22:39:32,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f. as already flushing 2024-11-10T22:39:32,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f. 2024-11-10T22:39:32,995 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => ccb0ca518fd65dd7264624eb4a78050f, NAME => 'TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:32,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => ccb0ca518fd65dd7264624eb4a78050f, NAME => 'TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:32,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ccb0ca518fd65dd7264624eb4a78050f, NAME => 'TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ccb0ca518fd65dd7264624eb4a78050f, NAME => 'TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:33,147 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-10T22:39:33,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f. 2024-11-10T22:39:33,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f. as already flushing 2024-11-10T22:39:33,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f. 2024-11-10T22:39:33,148 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => ccb0ca518fd65dd7264624eb4a78050f, NAME => 'TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:33,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => ccb0ca518fd65dd7264624eb4a78050f, NAME => 'TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:33,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ccb0ca518fd65dd7264624eb4a78050f, NAME => 'TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ccb0ca518fd65dd7264624eb4a78050f, NAME => 'TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:33,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-10T22:39:33,281 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:33,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42340 deadline: 1731278433272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:33,282 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:33,282 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:33,282 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:33,286 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:33,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42328 deadline: 1731278433282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:33,286 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:33,286 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:33,287 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:33,301 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-10T22:39:33,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f. 2024-11-10T22:39:33,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f. as already flushing 2024-11-10T22:39:33,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f. 2024-11-10T22:39:33,301 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => ccb0ca518fd65dd7264624eb4a78050f, NAME => 'TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:33,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => ccb0ca518fd65dd7264624eb4a78050f, NAME => 'TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:33,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ccb0ca518fd65dd7264624eb4a78050f, NAME => 'TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ccb0ca518fd65dd7264624eb4a78050f, NAME => 'TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:33,310 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:33,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42346 deadline: 1731278433303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:33,310 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:33,310 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:33,310 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:33,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42322 deadline: 1731278433303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:33,310 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:33,310 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:33,310 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:33,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42356 deadline: 1731278433303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:33,311 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:33,311 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:33,311 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:33,311 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:33,311 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:33,360 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/C/87d429300f2d47e59f9ffcf4de4d6a61 2024-11-10T22:39:33,364 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/A/279bda6940fd43d68f05566362ff9f78 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/A/279bda6940fd43d68f05566362ff9f78 2024-11-10T22:39:33,368 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/A/279bda6940fd43d68f05566362ff9f78, entries=200, sequenceid=14, filesize=14.0 K 2024-11-10T22:39:33,369 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/B/aa0700e4a9234c1b8380465d65e362c7 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/B/aa0700e4a9234c1b8380465d65e362c7 2024-11-10T22:39:33,372 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/B/aa0700e4a9234c1b8380465d65e362c7, entries=150, sequenceid=14, filesize=11.7 K 2024-11-10T22:39:33,373 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/C/87d429300f2d47e59f9ffcf4de4d6a61 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/C/87d429300f2d47e59f9ffcf4de4d6a61 2024-11-10T22:39:33,377 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/C/87d429300f2d47e59f9ffcf4de4d6a61, entries=150, sequenceid=14, filesize=11.7 K 2024-11-10T22:39:33,378 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=140.89 KB/144270 for ccb0ca518fd65dd7264624eb4a78050f in 1296ms, sequenceid=14, compaction requested=false 2024-11-10T22:39:33,378 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ccb0ca518fd65dd7264624eb4a78050f: 2024-11-10T22:39:33,453 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-10T22:39:33,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f. 2024-11-10T22:39:33,454 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2902): Flushing ccb0ca518fd65dd7264624eb4a78050f 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-10T22:39:33,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/A/2b54f76a241245999048e3f8c0df72ad is 50, key is test_row_0/A:col10/1731278372129/Put/seqid=0 2024-11-10T22:39:33,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742275_1451 (size=12001) 2024-11-10T22:39:33,863 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/A/2b54f76a241245999048e3f8c0df72ad 2024-11-10T22:39:33,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/B/c54800d2f8e44045b2c9356b70df9e50 is 50, key is test_row_0/B:col10/1731278372129/Put/seqid=0 2024-11-10T22:39:33,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742276_1452 (size=12001) 2024-11-10T22:39:34,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-10T22:39:34,274 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/B/c54800d2f8e44045b2c9356b70df9e50 2024-11-10T22:39:34,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/C/5fbd6182c0ee4b45b8388005d0d3de5a is 50, key is test_row_0/C:col10/1731278372129/Put/seqid=0 2024-11-10T22:39:34,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742277_1453 (size=12001) 2024-11-10T22:39:34,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(8855): Flush requested on ccb0ca518fd65dd7264624eb4a78050f 2024-11-10T22:39:34,298 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f. as already flushing 2024-11-10T22:39:34,326 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:34,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42346 deadline: 1731278434322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:34,327 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:34,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42322 deadline: 1731278434323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:34,327 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:34,327 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 849d0d71310f,43753,1731278250663 for 'test_row_0' in ccb0ca518fd65dd7264624eb4a78050f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2206 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-10T22:39:34,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42356 deadline: 1731278434323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:34,328 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:34,328 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:34,328 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:34,327 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 849d0d71310f,43753,1731278250663 for 'test_row_2' in ccb0ca518fd65dd7264624eb4a78050f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2198 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-10T22:39:34,328 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:34,328 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:34,328 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:34,328 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 849d0d71310f,43753,1731278250663 for 'test_row_0' in ccb0ca518fd65dd7264624eb4a78050f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2198 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-10T22:39:34,328 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:34,328 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:34,328 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:34,334 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:34,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42328 deadline: 1731278434324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:34,334 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:34,334 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:34,334 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:34,334 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:34,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42340 deadline: 1731278434326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:34,334 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:34,334 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:34,334 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:34,450 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:34,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42328 deadline: 1731278434443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:34,450 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:34,450 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:34,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42340 deadline: 1731278434443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:34,450 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:34,451 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:34,451 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:34,451 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:34,451 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:34,658 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:34,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42328 deadline: 1731278434652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:34,658 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:34,658 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:34,658 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:34,668 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:34,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42340 deadline: 1731278434663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:34,668 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:34,669 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:34,669 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:34,685 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/C/5fbd6182c0ee4b45b8388005d0d3de5a 2024-11-10T22:39:34,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/A/2b54f76a241245999048e3f8c0df72ad as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/A/2b54f76a241245999048e3f8c0df72ad 2024-11-10T22:39:34,692 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/A/2b54f76a241245999048e3f8c0df72ad, entries=150, sequenceid=38, filesize=11.7 K 2024-11-10T22:39:34,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/B/c54800d2f8e44045b2c9356b70df9e50 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/B/c54800d2f8e44045b2c9356b70df9e50 2024-11-10T22:39:34,696 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/B/c54800d2f8e44045b2c9356b70df9e50, entries=150, sequenceid=38, filesize=11.7 K 2024-11-10T22:39:34,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/C/5fbd6182c0ee4b45b8388005d0d3de5a as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/C/5fbd6182c0ee4b45b8388005d0d3de5a 2024-11-10T22:39:34,700 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/C/5fbd6182c0ee4b45b8388005d0d3de5a, entries=150, sequenceid=38, filesize=11.7 K 2024-11-10T22:39:34,701 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for ccb0ca518fd65dd7264624eb4a78050f in 1247ms, sequenceid=38, compaction requested=false 2024-11-10T22:39:34,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2603): Flush status journal for ccb0ca518fd65dd7264624eb4a78050f: 2024-11-10T22:39:34,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f. 2024-11-10T22:39:34,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=132 2024-11-10T22:39:34,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.HMaster(4169): Remote procedure done, pid=132 2024-11-10T22:39:34,704 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-11-10T22:39:34,704 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6260 sec 2024-11-10T22:39:34,706 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees in 2.6350 sec 2024-11-10T22:39:34,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(8855): Flush requested on ccb0ca518fd65dd7264624eb4a78050f 2024-11-10T22:39:34,968 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T22:39:34,968 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ccb0ca518fd65dd7264624eb4a78050f 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-10T22:39:34,973 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/A/47627124fced434189b9fb0d529b4b50 is 50, key is test_row_0/A:col10/1731278374323/Put/seqid=0 2024-11-10T22:39:34,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742278_1454 (size=14341) 2024-11-10T22:39:34,998 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-10T22:39:35,071 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:35,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42328 deadline: 1731278435058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:35,072 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:35,072 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:35,072 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:35,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42340 deadline: 1731278435060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:35,072 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:35,072 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:35,072 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:35,072 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:35,186 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:35,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42328 deadline: 1731278435182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:35,186 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:35,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42340 deadline: 1731278435182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:35,186 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:35,186 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:35,186 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:35,187 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:35,187 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:35,187 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:35,379 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/A/47627124fced434189b9fb0d529b4b50 2024-11-10T22:39:35,386 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/B/276a98a5b3b2416aa59dbc2fa649b324 is 50, key is test_row_0/B:col10/1731278374323/Put/seqid=0 2024-11-10T22:39:35,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742279_1455 (size=12001) 2024-11-10T22:39:35,392 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/B/276a98a5b3b2416aa59dbc2fa649b324 2024-11-10T22:39:35,398 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/C/a75f3b7fbeb14c5f80574cad7f8d9c1f is 50, key is test_row_0/C:col10/1731278374323/Put/seqid=0 2024-11-10T22:39:35,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742280_1456 (size=12001) 2024-11-10T22:39:35,403 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:35,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42340 deadline: 1731278435392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:35,403 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:35,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42328 deadline: 1731278435392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:35,404 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:35,404 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:35,404 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:35,406 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:35,406 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:35,406 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:35,722 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:35,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42340 deadline: 1731278435713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:35,722 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:35,722 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:35,722 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:35,722 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:35,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42328 deadline: 1731278435713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:35,723 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:35,723 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:35,723 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:35,803 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/C/a75f3b7fbeb14c5f80574cad7f8d9c1f 2024-11-10T22:39:35,807 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/A/47627124fced434189b9fb0d529b4b50 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/A/47627124fced434189b9fb0d529b4b50 2024-11-10T22:39:35,809 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/A/47627124fced434189b9fb0d529b4b50, entries=200, sequenceid=51, filesize=14.0 K 2024-11-10T22:39:35,810 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/B/276a98a5b3b2416aa59dbc2fa649b324 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/B/276a98a5b3b2416aa59dbc2fa649b324 2024-11-10T22:39:35,813 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/B/276a98a5b3b2416aa59dbc2fa649b324, entries=150, sequenceid=51, filesize=11.7 K 2024-11-10T22:39:35,814 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/C/a75f3b7fbeb14c5f80574cad7f8d9c1f as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/C/a75f3b7fbeb14c5f80574cad7f8d9c1f 2024-11-10T22:39:35,817 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/C/a75f3b7fbeb14c5f80574cad7f8d9c1f, entries=150, sequenceid=51, filesize=11.7 K 2024-11-10T22:39:35,818 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for ccb0ca518fd65dd7264624eb4a78050f in 850ms, sequenceid=51, compaction requested=true 2024-11-10T22:39:35,818 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ccb0ca518fd65dd7264624eb4a78050f: 2024-11-10T22:39:35,818 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ccb0ca518fd65dd7264624eb4a78050f:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T22:39:35,818 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T22:39:35,818 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ccb0ca518fd65dd7264624eb4a78050f:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T22:39:35,818 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T22:39:35,818 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T22:39:35,818 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ccb0ca518fd65dd7264624eb4a78050f:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T22:39:35,818 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T22:39:35,818 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T22:39:35,819 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T22:39:35,819 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40683 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T22:39:35,819 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HStore(1541): ccb0ca518fd65dd7264624eb4a78050f/A is initiating minor compaction (all files) 2024-11-10T22:39:35,819 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1541): ccb0ca518fd65dd7264624eb4a78050f/B is initiating minor compaction (all files) 2024-11-10T22:39:35,819 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ccb0ca518fd65dd7264624eb4a78050f/A in TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f. 2024-11-10T22:39:35,819 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ccb0ca518fd65dd7264624eb4a78050f/B in TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f. 2024-11-10T22:39:35,819 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/A/279bda6940fd43d68f05566362ff9f78, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/A/2b54f76a241245999048e3f8c0df72ad, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/A/47627124fced434189b9fb0d529b4b50] into tmpdir=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp, totalSize=39.7 K 2024-11-10T22:39:35,819 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/B/aa0700e4a9234c1b8380465d65e362c7, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/B/c54800d2f8e44045b2c9356b70df9e50, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/B/276a98a5b3b2416aa59dbc2fa649b324] into tmpdir=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp, totalSize=35.2 K 2024-11-10T22:39:35,819 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 279bda6940fd43d68f05566362ff9f78, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731278372079 2024-11-10T22:39:35,819 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting aa0700e4a9234c1b8380465d65e362c7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731278372079 2024-11-10T22:39:35,820 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting c54800d2f8e44045b2c9356b70df9e50, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1731278372118 2024-11-10T22:39:35,820 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2b54f76a241245999048e3f8c0df72ad, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1731278372118 2024-11-10T22:39:35,820 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting 276a98a5b3b2416aa59dbc2fa649b324, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731278374323 2024-11-10T22:39:35,820 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 47627124fced434189b9fb0d529b4b50, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731278374316 2024-11-10T22:39:35,826 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ccb0ca518fd65dd7264624eb4a78050f#B#compaction#377 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T22:39:35,826 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/B/bbcc8c5520574952a4cffb32a0653ba4 is 50, key is test_row_0/B:col10/1731278374323/Put/seqid=0 2024-11-10T22:39:35,826 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ccb0ca518fd65dd7264624eb4a78050f#A#compaction#378 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T22:39:35,827 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/A/b994f1f9ef4044b58412dabe05596407 is 50, key is test_row_0/A:col10/1731278374323/Put/seqid=0 2024-11-10T22:39:35,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742282_1458 (size=12104) 2024-11-10T22:39:35,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742281_1457 (size=12104) 2024-11-10T22:39:35,843 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/A/b994f1f9ef4044b58412dabe05596407 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/A/b994f1f9ef4044b58412dabe05596407 2024-11-10T22:39:35,843 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/B/bbcc8c5520574952a4cffb32a0653ba4 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/B/bbcc8c5520574952a4cffb32a0653ba4 2024-11-10T22:39:35,849 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ccb0ca518fd65dd7264624eb4a78050f/A of ccb0ca518fd65dd7264624eb4a78050f into b994f1f9ef4044b58412dabe05596407(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T22:39:35,849 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ccb0ca518fd65dd7264624eb4a78050f: 2024-11-10T22:39:35,849 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., storeName=ccb0ca518fd65dd7264624eb4a78050f/A, priority=13, startTime=1731278375818; duration=0sec 2024-11-10T22:39:35,849 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T22:39:35,849 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ccb0ca518fd65dd7264624eb4a78050f:A 2024-11-10T22:39:35,849 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ccb0ca518fd65dd7264624eb4a78050f/B of ccb0ca518fd65dd7264624eb4a78050f into bbcc8c5520574952a4cffb32a0653ba4(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T22:39:35,849 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T22:39:35,849 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ccb0ca518fd65dd7264624eb4a78050f: 2024-11-10T22:39:35,849 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., storeName=ccb0ca518fd65dd7264624eb4a78050f/B, priority=13, startTime=1731278375818; duration=0sec 2024-11-10T22:39:35,849 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T22:39:35,850 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ccb0ca518fd65dd7264624eb4a78050f:B 2024-11-10T22:39:35,850 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T22:39:35,850 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HStore(1541): ccb0ca518fd65dd7264624eb4a78050f/C is initiating minor compaction (all files) 2024-11-10T22:39:35,850 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ccb0ca518fd65dd7264624eb4a78050f/C in TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f. 2024-11-10T22:39:35,850 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/C/87d429300f2d47e59f9ffcf4de4d6a61, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/C/5fbd6182c0ee4b45b8388005d0d3de5a, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/C/a75f3b7fbeb14c5f80574cad7f8d9c1f] into tmpdir=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp, totalSize=35.2 K 2024-11-10T22:39:35,851 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 87d429300f2d47e59f9ffcf4de4d6a61, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731278372079 2024-11-10T22:39:35,851 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5fbd6182c0ee4b45b8388005d0d3de5a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1731278372118 2024-11-10T22:39:35,851 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting a75f3b7fbeb14c5f80574cad7f8d9c1f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731278374323 2024-11-10T22:39:35,858 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ccb0ca518fd65dd7264624eb4a78050f#C#compaction#379 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T22:39:35,858 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/C/82006ec117f64b21b03a7c6d03992925 is 50, key is test_row_0/C:col10/1731278374323/Put/seqid=0 2024-11-10T22:39:35,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742283_1459 (size=12104) 2024-11-10T22:39:36,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-10T22:39:36,233 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T22:39:36,234 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T22:39:36,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees 2024-11-10T22:39:36,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-10T22:39:36,235 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T22:39:36,236 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T22:39:36,236 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T22:39:36,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(8855): Flush requested on ccb0ca518fd65dd7264624eb4a78050f 2024-11-10T22:39:36,239 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T22:39:36,239 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ccb0ca518fd65dd7264624eb4a78050f 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-10T22:39:36,245 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/A/6acb83a744eb48678bb74cb2de683dec is 50, key is test_row_0/A:col10/1731278376237/Put/seqid=0 2024-11-10T22:39:36,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742284_1460 (size=16681) 2024-11-10T22:39:36,269 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/C/82006ec117f64b21b03a7c6d03992925 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/C/82006ec117f64b21b03a7c6d03992925 2024-11-10T22:39:36,272 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ccb0ca518fd65dd7264624eb4a78050f/C of ccb0ca518fd65dd7264624eb4a78050f into 82006ec117f64b21b03a7c6d03992925(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T22:39:36,272 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ccb0ca518fd65dd7264624eb4a78050f: 2024-11-10T22:39:36,272 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., storeName=ccb0ca518fd65dd7264624eb4a78050f/C, priority=13, startTime=1731278375818; duration=0sec 2024-11-10T22:39:36,272 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T22:39:36,273 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ccb0ca518fd65dd7264624eb4a78050f:C 2024-11-10T22:39:36,277 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:36,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42340 deadline: 1731278436267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:36,277 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:36,277 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:36,277 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:36,286 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:36,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42328 deadline: 1731278436276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:36,287 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:36,287 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:36,287 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:36,339 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:36,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42356 deadline: 1731278436332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:36,340 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 849d0d71310f,43753,1731278250663 for 'test_row_0' in ccb0ca518fd65dd7264624eb4a78050f of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4209 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-10T22:39:36,340 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:36,340 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:36,340 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:36,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-10T22:39:36,351 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:36,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42322 deadline: 1731278436342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:36,352 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 849d0d71310f,43753,1731278250663 for 'test_row_2' in ccb0ca518fd65dd7264624eb4a78050f of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4222 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-10T22:39:36,352 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:36,352 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:36,352 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:36,361 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:36,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42346 deadline: 1731278436352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:36,361 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 849d0d71310f,43753,1731278250663 for 'test_row_0' in ccb0ca518fd65dd7264624eb4a78050f of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4241 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-10T22:39:36,362 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:36,362 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:36,362 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:36,387 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:36,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42340 deadline: 1731278436383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:36,388 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:36,388 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:36,388 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:36,389 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-10T22:39:36,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f. 2024-11-10T22:39:36,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f. as already flushing 2024-11-10T22:39:36,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f. 2024-11-10T22:39:36,389 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => ccb0ca518fd65dd7264624eb4a78050f, NAME => 'TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:36,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => ccb0ca518fd65dd7264624eb4a78050f, NAME => 'TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:36,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ccb0ca518fd65dd7264624eb4a78050f, NAME => 'TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ccb0ca518fd65dd7264624eb4a78050f, NAME => 'TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:36,400 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:36,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42328 deadline: 1731278436392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:36,400 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:36,400 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:36,400 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:36,542 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-10T22:39:36,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f. 2024-11-10T22:39:36,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f. as already flushing 2024-11-10T22:39:36,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f. 2024-11-10T22:39:36,542 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => ccb0ca518fd65dd7264624eb4a78050f, NAME => 'TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:36,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => ccb0ca518fd65dd7264624eb4a78050f, NAME => 'TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:36,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ccb0ca518fd65dd7264624eb4a78050f, NAME => 'TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ccb0ca518fd65dd7264624eb4a78050f, NAME => 'TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:36,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-10T22:39:36,600 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:36,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42340 deadline: 1731278436592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:36,601 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:36,601 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:36,601 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:36,621 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:36,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42328 deadline: 1731278436612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:36,622 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:36,622 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:36,622 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:36,652 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/A/6acb83a744eb48678bb74cb2de683dec 2024-11-10T22:39:36,658 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/B/729255f0edd94ba59be3212284a86704 is 50, key is test_row_0/B:col10/1731278376237/Put/seqid=0 2024-11-10T22:39:36,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742285_1461 (size=12001) 2024-11-10T22:39:36,695 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-10T22:39:36,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f. 2024-11-10T22:39:36,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f. as already flushing 2024-11-10T22:39:36,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f. 2024-11-10T22:39:36,695 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => ccb0ca518fd65dd7264624eb4a78050f, NAME => 'TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:36,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => ccb0ca518fd65dd7264624eb4a78050f, NAME => 'TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:36,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ccb0ca518fd65dd7264624eb4a78050f, NAME => 'TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ccb0ca518fd65dd7264624eb4a78050f, NAME => 'TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:36,848 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-10T22:39:36,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f. 2024-11-10T22:39:36,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f. as already flushing 2024-11-10T22:39:36,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f. 2024-11-10T22:39:36,848 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => ccb0ca518fd65dd7264624eb4a78050f, NAME => 'TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:36,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => ccb0ca518fd65dd7264624eb4a78050f, NAME => 'TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:36,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ccb0ca518fd65dd7264624eb4a78050f, NAME => 'TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ccb0ca518fd65dd7264624eb4a78050f, NAME => 'TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:36,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-10T22:39:36,920 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:36,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42340 deadline: 1731278436912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:36,920 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:36,920 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:36,920 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:36,935 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:36,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42328 deadline: 1731278436932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:36,935 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:36,935 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:36,935 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:37,001 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-10T22:39:37,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f. 2024-11-10T22:39:37,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f. as already flushing 2024-11-10T22:39:37,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f. 2024-11-10T22:39:37,001 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => ccb0ca518fd65dd7264624eb4a78050f, NAME => 'TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:37,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => ccb0ca518fd65dd7264624eb4a78050f, NAME => 'TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:37,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ccb0ca518fd65dd7264624eb4a78050f, NAME => 'TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ccb0ca518fd65dd7264624eb4a78050f, NAME => 'TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:37,063 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/B/729255f0edd94ba59be3212284a86704 2024-11-10T22:39:37,070 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/C/c7af908f2d49400faef2fbde9d3037d0 is 50, key is test_row_0/C:col10/1731278376237/Put/seqid=0 2024-11-10T22:39:37,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742286_1462 (size=12001) 2024-11-10T22:39:37,154 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-10T22:39:37,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f. 2024-11-10T22:39:37,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f. as already flushing 2024-11-10T22:39:37,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f. 2024-11-10T22:39:37,154 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => ccb0ca518fd65dd7264624eb4a78050f, NAME => 'TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:37,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => ccb0ca518fd65dd7264624eb4a78050f, NAME => 'TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:37,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ccb0ca518fd65dd7264624eb4a78050f, NAME => 'TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ccb0ca518fd65dd7264624eb4a78050f, NAME => 'TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:37,307 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-10T22:39:37,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f. 2024-11-10T22:39:37,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f. as already flushing 2024-11-10T22:39:37,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f. 2024-11-10T22:39:37,307 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => ccb0ca518fd65dd7264624eb4a78050f, NAME => 'TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:37,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => ccb0ca518fd65dd7264624eb4a78050f, NAME => 'TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:37,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ccb0ca518fd65dd7264624eb4a78050f, NAME => 'TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ccb0ca518fd65dd7264624eb4a78050f, NAME => 'TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:37,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-10T22:39:37,438 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:37,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42340 deadline: 1731278437432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:37,438 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:37,438 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:37,438 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:37,450 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:37,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42328 deadline: 1731278437442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:37,451 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:37,451 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:37,451 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:37,460 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-10T22:39:37,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f. 2024-11-10T22:39:37,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f. as already flushing 2024-11-10T22:39:37,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f. 2024-11-10T22:39:37,460 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => ccb0ca518fd65dd7264624eb4a78050f, NAME => 'TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:37,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => ccb0ca518fd65dd7264624eb4a78050f, NAME => 'TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:37,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ccb0ca518fd65dd7264624eb4a78050f, NAME => 'TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ccb0ca518fd65dd7264624eb4a78050f, NAME => 'TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:37,474 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/C/c7af908f2d49400faef2fbde9d3037d0 2024-11-10T22:39:37,478 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/A/6acb83a744eb48678bb74cb2de683dec as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/A/6acb83a744eb48678bb74cb2de683dec 2024-11-10T22:39:37,481 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/A/6acb83a744eb48678bb74cb2de683dec, entries=250, sequenceid=78, filesize=16.3 K 2024-11-10T22:39:37,482 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/B/729255f0edd94ba59be3212284a86704 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/B/729255f0edd94ba59be3212284a86704 2024-11-10T22:39:37,484 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/B/729255f0edd94ba59be3212284a86704, entries=150, sequenceid=78, filesize=11.7 K 2024-11-10T22:39:37,485 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/C/c7af908f2d49400faef2fbde9d3037d0 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/C/c7af908f2d49400faef2fbde9d3037d0 2024-11-10T22:39:37,488 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/C/c7af908f2d49400faef2fbde9d3037d0, entries=150, sequenceid=78, filesize=11.7 K 2024-11-10T22:39:37,489 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for ccb0ca518fd65dd7264624eb4a78050f in 1249ms, sequenceid=78, compaction requested=false 2024-11-10T22:39:37,489 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ccb0ca518fd65dd7264624eb4a78050f: 2024-11-10T22:39:37,613 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-10T22:39:37,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f. 2024-11-10T22:39:37,613 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2902): Flushing ccb0ca518fd65dd7264624eb4a78050f 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-10T22:39:37,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/A/e7914d9aa25447c9a1e3cd08a5a1cad0 is 50, key is test_row_0/A:col10/1731278376265/Put/seqid=0 2024-11-10T22:39:37,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742287_1463 (size=12001) 2024-11-10T22:39:37,625 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/A/e7914d9aa25447c9a1e3cd08a5a1cad0 2024-11-10T22:39:37,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/B/0639e5ec3f0f4efc91e7367515c86882 is 50, key is test_row_0/B:col10/1731278376265/Put/seqid=0 2024-11-10T22:39:37,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742288_1464 (size=12001) 2024-11-10T22:39:38,040 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/B/0639e5ec3f0f4efc91e7367515c86882 2024-11-10T22:39:38,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/C/e237887b7a394e15b0e2418f929f60a3 is 50, key is test_row_0/C:col10/1731278376265/Put/seqid=0 2024-11-10T22:39:38,052 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742289_1465 (size=12001) 2024-11-10T22:39:38,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-10T22:39:38,453 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/C/e237887b7a394e15b0e2418f929f60a3 2024-11-10T22:39:38,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/A/e7914d9aa25447c9a1e3cd08a5a1cad0 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/A/e7914d9aa25447c9a1e3cd08a5a1cad0 2024-11-10T22:39:38,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(8855): Flush requested on ccb0ca518fd65dd7264624eb4a78050f 2024-11-10T22:39:38,458 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f. as already flushing 2024-11-10T22:39:38,464 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/A/e7914d9aa25447c9a1e3cd08a5a1cad0, entries=150, sequenceid=90, filesize=11.7 K 2024-11-10T22:39:38,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/B/0639e5ec3f0f4efc91e7367515c86882 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/B/0639e5ec3f0f4efc91e7367515c86882 2024-11-10T22:39:38,468 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/B/0639e5ec3f0f4efc91e7367515c86882, entries=150, sequenceid=90, filesize=11.7 K 2024-11-10T22:39:38,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/C/e237887b7a394e15b0e2418f929f60a3 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/C/e237887b7a394e15b0e2418f929f60a3 2024-11-10T22:39:38,477 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/C/e237887b7a394e15b0e2418f929f60a3, entries=150, sequenceid=90, filesize=11.7 K 2024-11-10T22:39:38,478 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=33.54 KB/34350 for ccb0ca518fd65dd7264624eb4a78050f in 865ms, sequenceid=90, compaction requested=true 2024-11-10T22:39:38,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2603): Flush status journal for ccb0ca518fd65dd7264624eb4a78050f: 2024-11-10T22:39:38,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f. 2024-11-10T22:39:38,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=134 2024-11-10T22:39:38,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.HMaster(4169): Remote procedure done, pid=134 2024-11-10T22:39:38,481 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-11-10T22:39:38,481 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2440 sec 2024-11-10T22:39:38,483 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees in 2.2480 sec 2024-11-10T22:39:38,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(8855): Flush requested on ccb0ca518fd65dd7264624eb4a78050f 2024-11-10T22:39:38,486 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T22:39:38,486 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ccb0ca518fd65dd7264624eb4a78050f 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-10T22:39:38,491 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/A/0bacdee9eb9b4185bd5175357adc09e6 is 50, key is test_row_0/A:col10/1731278378483/Put/seqid=0 2024-11-10T22:39:38,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742290_1466 (size=14341) 2024-11-10T22:39:38,498 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=101 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/A/0bacdee9eb9b4185bd5175357adc09e6 2024-11-10T22:39:38,504 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/B/f81cc73ebbea452eb141b8e8f15599bc is 50, key is test_row_0/B:col10/1731278378483/Put/seqid=0 2024-11-10T22:39:38,511 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742291_1467 (size=12001) 2024-11-10T22:39:38,511 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=101 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/B/f81cc73ebbea452eb141b8e8f15599bc 2024-11-10T22:39:38,519 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/C/f24fc557f2cb4b63aef92b6f3a678c8c is 50, key is test_row_0/C:col10/1731278378483/Put/seqid=0 2024-11-10T22:39:38,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742292_1468 (size=12001) 2024-11-10T22:39:38,631 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:38,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42340 deadline: 1731278438617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:38,632 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:38,632 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:38,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42328 deadline: 1731278438618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:38,632 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:38,632 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:38,632 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:38,632 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:38,632 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:38,752 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:38,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42340 deadline: 1731278438742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:38,752 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:38,752 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:38,752 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:38,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42328 deadline: 1731278438742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:38,752 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:38,753 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:38,753 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:38,753 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:38,923 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=101 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/C/f24fc557f2cb4b63aef92b6f3a678c8c 2024-11-10T22:39:38,926 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/A/0bacdee9eb9b4185bd5175357adc09e6 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/A/0bacdee9eb9b4185bd5175357adc09e6 2024-11-10T22:39:38,929 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/A/0bacdee9eb9b4185bd5175357adc09e6, entries=200, sequenceid=101, filesize=14.0 K 2024-11-10T22:39:38,930 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/B/f81cc73ebbea452eb141b8e8f15599bc as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/B/f81cc73ebbea452eb141b8e8f15599bc 2024-11-10T22:39:38,933 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/B/f81cc73ebbea452eb141b8e8f15599bc, entries=150, sequenceid=101, filesize=11.7 K 2024-11-10T22:39:38,933 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/C/f24fc557f2cb4b63aef92b6f3a678c8c as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/C/f24fc557f2cb4b63aef92b6f3a678c8c 2024-11-10T22:39:38,936 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/C/f24fc557f2cb4b63aef92b6f3a678c8c, entries=150, sequenceid=101, filesize=11.7 K 2024-11-10T22:39:38,937 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for ccb0ca518fd65dd7264624eb4a78050f in 451ms, sequenceid=101, compaction requested=true 2024-11-10T22:39:38,937 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ccb0ca518fd65dd7264624eb4a78050f: 2024-11-10T22:39:38,937 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ccb0ca518fd65dd7264624eb4a78050f:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T22:39:38,937 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T22:39:38,937 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T22:39:38,937 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ccb0ca518fd65dd7264624eb4a78050f:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T22:39:38,937 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T22:39:38,937 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ccb0ca518fd65dd7264624eb4a78050f:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T22:39:38,937 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T22:39:38,937 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T22:39:38,940 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T22:39:38,940 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 55127 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T22:39:38,940 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1541): ccb0ca518fd65dd7264624eb4a78050f/B is initiating minor compaction (all files) 2024-11-10T22:39:38,940 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HStore(1541): ccb0ca518fd65dd7264624eb4a78050f/A is initiating minor compaction (all files) 2024-11-10T22:39:38,940 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ccb0ca518fd65dd7264624eb4a78050f/A in TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f. 2024-11-10T22:39:38,940 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ccb0ca518fd65dd7264624eb4a78050f/B in TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f. 2024-11-10T22:39:38,940 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/B/bbcc8c5520574952a4cffb32a0653ba4, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/B/729255f0edd94ba59be3212284a86704, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/B/0639e5ec3f0f4efc91e7367515c86882, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/B/f81cc73ebbea452eb141b8e8f15599bc] into tmpdir=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp, totalSize=47.0 K 2024-11-10T22:39:38,940 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/A/b994f1f9ef4044b58412dabe05596407, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/A/6acb83a744eb48678bb74cb2de683dec, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/A/e7914d9aa25447c9a1e3cd08a5a1cad0, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/A/0bacdee9eb9b4185bd5175357adc09e6] into tmpdir=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp, totalSize=53.8 K 2024-11-10T22:39:38,941 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting b994f1f9ef4044b58412dabe05596407, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731278374323 2024-11-10T22:39:38,941 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting bbcc8c5520574952a4cffb32a0653ba4, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731278374323 2024-11-10T22:39:38,941 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6acb83a744eb48678bb74cb2de683dec, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1731278375051 2024-11-10T22:39:38,941 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting 729255f0edd94ba59be3212284a86704, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1731278375051 2024-11-10T22:39:38,941 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting e7914d9aa25447c9a1e3cd08a5a1cad0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1731278376256 2024-11-10T22:39:38,941 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting 0639e5ec3f0f4efc91e7367515c86882, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1731278376256 2024-11-10T22:39:38,941 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0bacdee9eb9b4185bd5175357adc09e6, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1731278378465 2024-11-10T22:39:38,941 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting f81cc73ebbea452eb141b8e8f15599bc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1731278378465 2024-11-10T22:39:38,948 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ccb0ca518fd65dd7264624eb4a78050f#B#compaction#389 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T22:39:38,948 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/B/e8655bdb107c4710b0999f7d4e50bea5 is 50, key is test_row_0/B:col10/1731278378483/Put/seqid=0 2024-11-10T22:39:38,951 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ccb0ca518fd65dd7264624eb4a78050f#A#compaction#390 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T22:39:38,952 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742293_1469 (size=12241) 2024-11-10T22:39:38,952 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/A/c3a460221f4648f686db1bcb736c1cf9 is 50, key is test_row_0/A:col10/1731278378483/Put/seqid=0 2024-11-10T22:39:38,957 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/B/e8655bdb107c4710b0999f7d4e50bea5 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/B/e8655bdb107c4710b0999f7d4e50bea5 2024-11-10T22:39:38,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742294_1470 (size=12241) 2024-11-10T22:39:38,961 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ccb0ca518fd65dd7264624eb4a78050f/B of ccb0ca518fd65dd7264624eb4a78050f into e8655bdb107c4710b0999f7d4e50bea5(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T22:39:38,961 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ccb0ca518fd65dd7264624eb4a78050f: 2024-11-10T22:39:38,961 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., storeName=ccb0ca518fd65dd7264624eb4a78050f/B, priority=12, startTime=1731278378937; duration=0sec 2024-11-10T22:39:38,961 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T22:39:38,961 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ccb0ca518fd65dd7264624eb4a78050f:B 2024-11-10T22:39:38,961 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T22:39:38,963 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T22:39:38,963 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1541): ccb0ca518fd65dd7264624eb4a78050f/C is initiating minor compaction (all files) 2024-11-10T22:39:38,964 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ccb0ca518fd65dd7264624eb4a78050f/C in TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f. 2024-11-10T22:39:38,964 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/C/82006ec117f64b21b03a7c6d03992925, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/C/c7af908f2d49400faef2fbde9d3037d0, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/C/e237887b7a394e15b0e2418f929f60a3, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/C/f24fc557f2cb4b63aef92b6f3a678c8c] into tmpdir=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp, totalSize=47.0 K 2024-11-10T22:39:38,964 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting 82006ec117f64b21b03a7c6d03992925, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731278374323 2024-11-10T22:39:38,964 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting c7af908f2d49400faef2fbde9d3037d0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1731278375051 2024-11-10T22:39:38,964 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting e237887b7a394e15b0e2418f929f60a3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1731278376256 2024-11-10T22:39:38,965 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting f24fc557f2cb4b63aef92b6f3a678c8c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1731278378465 2024-11-10T22:39:38,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(8855): Flush requested on ccb0ca518fd65dd7264624eb4a78050f 2024-11-10T22:39:38,966 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T22:39:38,967 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ccb0ca518fd65dd7264624eb4a78050f 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-10T22:39:38,971 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/A/c520cf2332b84164a1760bd4108814fc is 50, key is test_row_0/A:col10/1731278378581/Put/seqid=0 2024-11-10T22:39:38,972 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ccb0ca518fd65dd7264624eb4a78050f#C#compaction#392 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T22:39:38,973 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/C/63a7980c1c7344d1be2f91cdf247ccde is 50, key is test_row_0/C:col10/1731278378483/Put/seqid=0 2024-11-10T22:39:38,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742295_1471 (size=14341) 2024-11-10T22:39:38,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742296_1472 (size=12241) 2024-11-10T22:39:39,002 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:39,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42340 deadline: 1731278438994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:39,003 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:39,003 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:39,003 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:39,007 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:39,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42328 deadline: 1731278439002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:39,007 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:39,007 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:39,007 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:39,118 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:39,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42340 deadline: 1731278439112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:39,119 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:39,119 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:39,119 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:39,119 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:39,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42328 deadline: 1731278439112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:39,119 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:39,119 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:39,119 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:39,330 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:39,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42340 deadline: 1731278439322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:39,330 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:39,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42328 deadline: 1731278439322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:39,330 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:39,331 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:39,331 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:39,331 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:39,331 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:39,331 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:39,363 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/A/c3a460221f4648f686db1bcb736c1cf9 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/A/c3a460221f4648f686db1bcb736c1cf9 2024-11-10T22:39:39,367 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ccb0ca518fd65dd7264624eb4a78050f/A of ccb0ca518fd65dd7264624eb4a78050f into c3a460221f4648f686db1bcb736c1cf9(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T22:39:39,367 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ccb0ca518fd65dd7264624eb4a78050f: 2024-11-10T22:39:39,367 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., storeName=ccb0ca518fd65dd7264624eb4a78050f/A, priority=12, startTime=1731278378937; duration=0sec 2024-11-10T22:39:39,367 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T22:39:39,367 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ccb0ca518fd65dd7264624eb4a78050f:A 2024-11-10T22:39:39,376 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=128 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/A/c520cf2332b84164a1760bd4108814fc 2024-11-10T22:39:39,380 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/C/63a7980c1c7344d1be2f91cdf247ccde as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/C/63a7980c1c7344d1be2f91cdf247ccde 2024-11-10T22:39:39,383 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/B/034ec481bc0c46d4bb8a1c26b4c7e01d is 50, key is test_row_0/B:col10/1731278378581/Put/seqid=0 2024-11-10T22:39:39,387 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ccb0ca518fd65dd7264624eb4a78050f/C of ccb0ca518fd65dd7264624eb4a78050f into 63a7980c1c7344d1be2f91cdf247ccde(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T22:39:39,387 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ccb0ca518fd65dd7264624eb4a78050f: 2024-11-10T22:39:39,387 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., storeName=ccb0ca518fd65dd7264624eb4a78050f/C, priority=12, startTime=1731278378937; duration=0sec 2024-11-10T22:39:39,387 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T22:39:39,387 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ccb0ca518fd65dd7264624eb4a78050f:C 2024-11-10T22:39:39,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742297_1473 (size=12001) 2024-11-10T22:39:39,649 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:39,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42340 deadline: 1731278439642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:39,649 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:39,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42328 deadline: 1731278439642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:39,649 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:39,649 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:39,649 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:39,650 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:39,650 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:39,650 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:39,789 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=128 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/B/034ec481bc0c46d4bb8a1c26b4c7e01d 2024-11-10T22:39:39,796 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/C/53267623b9c44abfbae788aaf65f6df2 is 50, key is test_row_0/C:col10/1731278378581/Put/seqid=0 2024-11-10T22:39:39,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742298_1474 (size=12001) 2024-11-10T22:39:40,160 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:40,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42340 deadline: 1731278440152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:40,160 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:40,160 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:40,160 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:40,172 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:40,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42328 deadline: 1731278440162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:40,172 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:40,173 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:40,173 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:40,200 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=128 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/C/53267623b9c44abfbae788aaf65f6df2 2024-11-10T22:39:40,204 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/A/c520cf2332b84164a1760bd4108814fc as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/A/c520cf2332b84164a1760bd4108814fc 2024-11-10T22:39:40,207 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/A/c520cf2332b84164a1760bd4108814fc, entries=200, sequenceid=128, filesize=14.0 K 2024-11-10T22:39:40,207 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/B/034ec481bc0c46d4bb8a1c26b4c7e01d as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/B/034ec481bc0c46d4bb8a1c26b4c7e01d 2024-11-10T22:39:40,211 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/B/034ec481bc0c46d4bb8a1c26b4c7e01d, entries=150, sequenceid=128, filesize=11.7 K 2024-11-10T22:39:40,211 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/C/53267623b9c44abfbae788aaf65f6df2 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/C/53267623b9c44abfbae788aaf65f6df2 2024-11-10T22:39:40,215 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/C/53267623b9c44abfbae788aaf65f6df2, entries=150, sequenceid=128, filesize=11.7 K 2024-11-10T22:39:40,215 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for ccb0ca518fd65dd7264624eb4a78050f in 1249ms, sequenceid=128, compaction requested=false 2024-11-10T22:39:40,215 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ccb0ca518fd65dd7264624eb4a78050f: 2024-11-10T22:39:40,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(8855): Flush requested on ccb0ca518fd65dd7264624eb4a78050f 2024-11-10T22:39:40,359 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T22:39:40,359 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ccb0ca518fd65dd7264624eb4a78050f 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-10T22:39:40,363 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/A/9bc522c3d9b447399cce6c192ac3d293 is 50, key is test_row_0/A:col10/1731278380357/Put/seqid=0 2024-11-10T22:39:40,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742299_1475 (size=14541) 2024-11-10T22:39:40,367 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=141 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/A/9bc522c3d9b447399cce6c192ac3d293 2024-11-10T22:39:40,373 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/B/3cc1419f79fa4f74b537a8897d30b4fe is 50, key is test_row_0/B:col10/1731278380357/Put/seqid=0 2024-11-10T22:39:40,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742300_1476 (size=12151) 2024-11-10T22:39:40,392 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-10T22:39:40,392 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-10T22:39:40,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-10T22:39:40,403 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T22:39:40,403 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T22:39:40,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] procedure2.ProcedureExecutor(1139): Stored pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees 2024-11-10T22:39:40,405 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T22:39:40,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-10T22:39:40,405 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T22:39:40,405 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T22:39:40,463 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:40,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42346 deadline: 1731278440450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:40,464 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:40,464 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:40,464 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:40,464 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:40,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42356 deadline: 1731278440453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:40,464 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:40,464 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:40,464 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:40,466 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:40,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42322 deadline: 1731278440458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:40,466 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:40,466 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:40,467 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:40,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-10T22:39:40,557 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-10T22:39:40,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f. 2024-11-10T22:39:40,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f. as already flushing 2024-11-10T22:39:40,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f. 2024-11-10T22:39:40,558 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => ccb0ca518fd65dd7264624eb4a78050f, NAME => 'TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:40,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => ccb0ca518fd65dd7264624eb4a78050f, NAME => 'TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:40,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ccb0ca518fd65dd7264624eb4a78050f, NAME => 'TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ccb0ca518fd65dd7264624eb4a78050f, NAME => 'TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:40,582 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:40,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42346 deadline: 1731278440573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:40,582 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:40,582 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:40,582 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:40,582 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:40,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42356 deadline: 1731278440573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:40,583 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:40,583 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:40,583 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:40,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42322 deadline: 1731278440573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:40,583 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:40,583 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:40,583 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:40,583 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:40,710 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-10T22:39:40,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f. 2024-11-10T22:39:40,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f. as already flushing 2024-11-10T22:39:40,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f. 2024-11-10T22:39:40,711 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => ccb0ca518fd65dd7264624eb4a78050f, NAME => 'TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:40,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => ccb0ca518fd65dd7264624eb4a78050f, NAME => 'TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:40,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ccb0ca518fd65dd7264624eb4a78050f, NAME => 'TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ccb0ca518fd65dd7264624eb4a78050f, NAME => 'TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:40,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-10T22:39:40,777 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=141 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/B/3cc1419f79fa4f74b537a8897d30b4fe 2024-11-10T22:39:40,785 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/C/d5759639fde545c0a08ad1f2d005f6ef is 50, key is test_row_0/C:col10/1731278380357/Put/seqid=0 2024-11-10T22:39:40,788 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742301_1477 (size=12151) 2024-11-10T22:39:40,798 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:40,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42346 deadline: 1731278440793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:40,798 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:40,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42356 deadline: 1731278440793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:40,798 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:40,798 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:40,799 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:40,799 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:40,799 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:40,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42322 deadline: 1731278440793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:40,799 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:40,799 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:40,799 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:40,799 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:40,799 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:40,863 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-10T22:39:40,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f. 2024-11-10T22:39:40,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f. as already flushing 2024-11-10T22:39:40,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f. 2024-11-10T22:39:40,864 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => ccb0ca518fd65dd7264624eb4a78050f, NAME => 'TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:40,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => ccb0ca518fd65dd7264624eb4a78050f, NAME => 'TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:40,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ccb0ca518fd65dd7264624eb4a78050f, NAME => 'TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ccb0ca518fd65dd7264624eb4a78050f, NAME => 'TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:41,016 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-10T22:39:41,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f. 2024-11-10T22:39:41,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f. as already flushing 2024-11-10T22:39:41,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f. 2024-11-10T22:39:41,017 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => ccb0ca518fd65dd7264624eb4a78050f, NAME => 'TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:41,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => ccb0ca518fd65dd7264624eb4a78050f, NAME => 'TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:41,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ccb0ca518fd65dd7264624eb4a78050f, NAME => 'TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ccb0ca518fd65dd7264624eb4a78050f, NAME => 'TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:41,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-10T22:39:41,112 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:41,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42356 deadline: 1731278441102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:41,112 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:41,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42322 deadline: 1731278441102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:41,113 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:41,113 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:41,113 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:41,113 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:41,113 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:41,113 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:41,121 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:41,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42346 deadline: 1731278441113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:41,121 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:41,121 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:41,121 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:41,170 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-10T22:39:41,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f. 2024-11-10T22:39:41,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f. as already flushing 2024-11-10T22:39:41,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f. 2024-11-10T22:39:41,170 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => ccb0ca518fd65dd7264624eb4a78050f, NAME => 'TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:41,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => ccb0ca518fd65dd7264624eb4a78050f, NAME => 'TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:41,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ccb0ca518fd65dd7264624eb4a78050f, NAME => 'TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ccb0ca518fd65dd7264624eb4a78050f, NAME => 'TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:41,174 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:41,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42340 deadline: 1731278441172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:41,175 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 849d0d71310f,43753,1731278250663 for 'test_row_2' in ccb0ca518fd65dd7264624eb4a78050f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2180 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-10T22:39:41,175 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:41,175 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:41,175 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:41,187 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:41,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42328 deadline: 1731278441182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:41,187 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 849d0d71310f,43753,1731278250663 for 'test_row_1' in ccb0ca518fd65dd7264624eb4a78050f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2185 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-10T22:39:41,188 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:41,188 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:41,188 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:41,189 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=141 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/C/d5759639fde545c0a08ad1f2d005f6ef 2024-11-10T22:39:41,193 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/A/9bc522c3d9b447399cce6c192ac3d293 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/A/9bc522c3d9b447399cce6c192ac3d293 2024-11-10T22:39:41,196 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/A/9bc522c3d9b447399cce6c192ac3d293, entries=200, sequenceid=141, filesize=14.2 K 2024-11-10T22:39:41,197 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/B/3cc1419f79fa4f74b537a8897d30b4fe as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/B/3cc1419f79fa4f74b537a8897d30b4fe 2024-11-10T22:39:41,200 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/B/3cc1419f79fa4f74b537a8897d30b4fe, entries=150, sequenceid=141, filesize=11.9 K 2024-11-10T22:39:41,201 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/C/d5759639fde545c0a08ad1f2d005f6ef as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/C/d5759639fde545c0a08ad1f2d005f6ef 2024-11-10T22:39:41,205 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/C/d5759639fde545c0a08ad1f2d005f6ef, entries=150, sequenceid=141, filesize=11.9 K 2024-11-10T22:39:41,206 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for ccb0ca518fd65dd7264624eb4a78050f in 847ms, sequenceid=141, compaction requested=true 2024-11-10T22:39:41,206 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ccb0ca518fd65dd7264624eb4a78050f: 2024-11-10T22:39:41,207 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ccb0ca518fd65dd7264624eb4a78050f:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T22:39:41,207 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T22:39:41,207 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T22:39:41,207 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T22:39:41,207 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ccb0ca518fd65dd7264624eb4a78050f:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T22:39:41,207 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T22:39:41,207 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ccb0ca518fd65dd7264624eb4a78050f:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T22:39:41,207 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T22:39:41,207 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41123 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T22:39:41,208 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36393 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T22:39:41,208 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HStore(1541): ccb0ca518fd65dd7264624eb4a78050f/A is initiating minor compaction (all files) 2024-11-10T22:39:41,208 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1541): ccb0ca518fd65dd7264624eb4a78050f/B is initiating minor compaction (all files) 2024-11-10T22:39:41,208 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ccb0ca518fd65dd7264624eb4a78050f/A in TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f. 2024-11-10T22:39:41,208 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ccb0ca518fd65dd7264624eb4a78050f/B in TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f. 2024-11-10T22:39:41,208 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/A/c3a460221f4648f686db1bcb736c1cf9, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/A/c520cf2332b84164a1760bd4108814fc, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/A/9bc522c3d9b447399cce6c192ac3d293] into tmpdir=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp, totalSize=40.2 K 2024-11-10T22:39:41,208 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/B/e8655bdb107c4710b0999f7d4e50bea5, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/B/034ec481bc0c46d4bb8a1c26b4c7e01d, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/B/3cc1419f79fa4f74b537a8897d30b4fe] into tmpdir=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp, totalSize=35.5 K 2024-11-10T22:39:41,208 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting c3a460221f4648f686db1bcb736c1cf9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1731278378465 2024-11-10T22:39:41,208 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting e8655bdb107c4710b0999f7d4e50bea5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1731278378465 2024-11-10T22:39:41,208 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting c520cf2332b84164a1760bd4108814fc, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1731278378581 2024-11-10T22:39:41,209 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting 034ec481bc0c46d4bb8a1c26b4c7e01d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1731278378581 2024-11-10T22:39:41,209 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting 3cc1419f79fa4f74b537a8897d30b4fe, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=141, earliestPutTs=1731278378980 2024-11-10T22:39:41,209 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9bc522c3d9b447399cce6c192ac3d293, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=141, earliestPutTs=1731278378980 2024-11-10T22:39:41,216 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ccb0ca518fd65dd7264624eb4a78050f#A#compaction#398 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T22:39:41,216 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/A/6e237ee03183468cb7cf9f8d95f49742 is 50, key is test_row_0/A:col10/1731278380357/Put/seqid=0 2024-11-10T22:39:41,218 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ccb0ca518fd65dd7264624eb4a78050f#B#compaction#399 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T22:39:41,218 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/B/234c945f6b004debad9f25609573ee8a is 50, key is test_row_0/B:col10/1731278380357/Put/seqid=0 2024-11-10T22:39:41,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742302_1478 (size=12493) 2024-11-10T22:39:41,224 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/A/6e237ee03183468cb7cf9f8d95f49742 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/A/6e237ee03183468cb7cf9f8d95f49742 2024-11-10T22:39:41,228 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ccb0ca518fd65dd7264624eb4a78050f/A of ccb0ca518fd65dd7264624eb4a78050f into 6e237ee03183468cb7cf9f8d95f49742(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T22:39:41,228 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ccb0ca518fd65dd7264624eb4a78050f: 2024-11-10T22:39:41,228 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., storeName=ccb0ca518fd65dd7264624eb4a78050f/A, priority=13, startTime=1731278381206; duration=0sec 2024-11-10T22:39:41,228 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T22:39:41,228 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ccb0ca518fd65dd7264624eb4a78050f:A 2024-11-10T22:39:41,228 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T22:39:41,229 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36393 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T22:39:41,229 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HStore(1541): ccb0ca518fd65dd7264624eb4a78050f/C is initiating minor compaction (all files) 2024-11-10T22:39:41,229 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ccb0ca518fd65dd7264624eb4a78050f/C in TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f. 2024-11-10T22:39:41,229 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/C/63a7980c1c7344d1be2f91cdf247ccde, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/C/53267623b9c44abfbae788aaf65f6df2, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/C/d5759639fde545c0a08ad1f2d005f6ef] into tmpdir=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp, totalSize=35.5 K 2024-11-10T22:39:41,229 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 63a7980c1c7344d1be2f91cdf247ccde, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1731278378465 2024-11-10T22:39:41,230 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 53267623b9c44abfbae788aaf65f6df2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1731278378581 2024-11-10T22:39:41,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742303_1479 (size=12493) 2024-11-10T22:39:41,230 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting d5759639fde545c0a08ad1f2d005f6ef, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=141, earliestPutTs=1731278378980 2024-11-10T22:39:41,234 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/B/234c945f6b004debad9f25609573ee8a as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/B/234c945f6b004debad9f25609573ee8a 2024-11-10T22:39:41,237 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ccb0ca518fd65dd7264624eb4a78050f#C#compaction#400 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T22:39:41,237 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/C/249507aa08374e0d87ad7b5e55ccaf46 is 50, key is test_row_0/C:col10/1731278380357/Put/seqid=0 2024-11-10T22:39:41,238 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ccb0ca518fd65dd7264624eb4a78050f/B of ccb0ca518fd65dd7264624eb4a78050f into 234c945f6b004debad9f25609573ee8a(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T22:39:41,238 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ccb0ca518fd65dd7264624eb4a78050f: 2024-11-10T22:39:41,238 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., storeName=ccb0ca518fd65dd7264624eb4a78050f/B, priority=13, startTime=1731278381207; duration=0sec 2024-11-10T22:39:41,238 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T22:39:41,238 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ccb0ca518fd65dd7264624eb4a78050f:B 2024-11-10T22:39:41,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742304_1480 (size=12493) 2024-11-10T22:39:41,245 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/C/249507aa08374e0d87ad7b5e55ccaf46 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/C/249507aa08374e0d87ad7b5e55ccaf46 2024-11-10T22:39:41,249 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ccb0ca518fd65dd7264624eb4a78050f/C of ccb0ca518fd65dd7264624eb4a78050f into 249507aa08374e0d87ad7b5e55ccaf46(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T22:39:41,249 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ccb0ca518fd65dd7264624eb4a78050f: 2024-11-10T22:39:41,249 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., storeName=ccb0ca518fd65dd7264624eb4a78050f/C, priority=13, startTime=1731278381207; duration=0sec 2024-11-10T22:39:41,249 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T22:39:41,249 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ccb0ca518fd65dd7264624eb4a78050f:C 2024-11-10T22:39:41,323 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-10T22:39:41,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f. 2024-11-10T22:39:41,324 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2902): Flushing ccb0ca518fd65dd7264624eb4a78050f 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-10T22:39:41,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/A/4f39b2de2ad24eec956bce0b5442f63c is 50, key is test_row_0/A:col10/1731278380452/Put/seqid=0 2024-11-10T22:39:41,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742305_1481 (size=12151) 2024-11-10T22:39:41,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-10T22:39:41,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(8855): Flush requested on ccb0ca518fd65dd7264624eb4a78050f 2024-11-10T22:39:41,626 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f. as already flushing 2024-11-10T22:39:41,647 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:41,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42356 deadline: 1731278441642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:41,648 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:41,648 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:41,648 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:41,653 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:41,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42322 deadline: 1731278441643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:41,654 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:41,654 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:41,654 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:41,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42346 deadline: 1731278441647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:41,654 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:41,654 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:41,654 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:41,654 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:41,733 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/A/4f39b2de2ad24eec956bce0b5442f63c 2024-11-10T22:39:41,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/B/ed048410aa224771829b127492153484 is 50, key is test_row_0/B:col10/1731278380452/Put/seqid=0 2024-11-10T22:39:41,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742306_1482 (size=12151) 2024-11-10T22:39:41,758 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:41,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42356 deadline: 1731278441752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:41,758 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:41,758 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:41,758 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:41,770 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:41,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42322 deadline: 1731278441762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:41,770 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:41,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42346 deadline: 1731278441763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:41,770 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:41,770 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:41,770 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:41,770 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:41,770 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:41,770 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:41,969 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:41,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42356 deadline: 1731278441962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:41,970 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:41,970 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:41,970 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:41,981 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:41,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42322 deadline: 1731278441972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:41,981 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:41,982 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:41,982 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:41,986 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:41,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42346 deadline: 1731278441982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:41,987 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:41,987 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:41,987 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:42,144 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/B/ed048410aa224771829b127492153484 2024-11-10T22:39:42,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/C/d02b83d77ad441d9b838760590ef8caa is 50, key is test_row_0/C:col10/1731278380452/Put/seqid=0 2024-11-10T22:39:42,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742307_1483 (size=12151) 2024-11-10T22:39:42,290 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:42,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42356 deadline: 1731278442283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:42,290 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:42,290 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:42,290 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:42,302 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:42,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42322 deadline: 1731278442292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:42,302 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:42,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42346 deadline: 1731278442292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:42,302 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:42,303 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:42,303 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:42,303 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:42,303 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:42,303 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:42,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-10T22:39:42,555 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/C/d02b83d77ad441d9b838760590ef8caa 2024-11-10T22:39:42,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/A/4f39b2de2ad24eec956bce0b5442f63c as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/A/4f39b2de2ad24eec956bce0b5442f63c 2024-11-10T22:39:42,563 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/A/4f39b2de2ad24eec956bce0b5442f63c, entries=150, sequenceid=169, filesize=11.9 K 2024-11-10T22:39:42,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/B/ed048410aa224771829b127492153484 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/B/ed048410aa224771829b127492153484 2024-11-10T22:39:42,568 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/B/ed048410aa224771829b127492153484, entries=150, sequenceid=169, filesize=11.9 K 2024-11-10T22:39:42,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/C/d02b83d77ad441d9b838760590ef8caa as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/C/d02b83d77ad441d9b838760590ef8caa 2024-11-10T22:39:42,572 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/C/d02b83d77ad441d9b838760590ef8caa, entries=150, sequenceid=169, filesize=11.9 K 2024-11-10T22:39:42,573 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for ccb0ca518fd65dd7264624eb4a78050f in 1250ms, sequenceid=169, compaction requested=false 2024-11-10T22:39:42,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2603): Flush status journal for ccb0ca518fd65dd7264624eb4a78050f: 2024-11-10T22:39:42,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f. 2024-11-10T22:39:42,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=136 2024-11-10T22:39:42,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.HMaster(4169): Remote procedure done, pid=136 2024-11-10T22:39:42,576 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-11-10T22:39:42,577 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1690 sec 2024-11-10T22:39:42,578 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees in 2.1740 sec 2024-11-10T22:39:42,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(8855): Flush requested on ccb0ca518fd65dd7264624eb4a78050f 2024-11-10T22:39:42,807 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T22:39:42,807 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ccb0ca518fd65dd7264624eb4a78050f 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-10T22:39:42,811 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/A/341c7132a0954a5294c7009c62c4b5f6 is 50, key is test_row_0/A:col10/1731278381645/Put/seqid=0 2024-11-10T22:39:42,815 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742308_1484 (size=14541) 2024-11-10T22:39:42,892 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:42,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42356 deadline: 1731278442881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:42,892 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:42,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42322 deadline: 1731278442882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:42,892 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:42,892 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:42,893 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:42,893 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:42,893 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:42,893 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:42,894 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:42,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42346 deadline: 1731278442883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:42,895 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:42,895 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:42,895 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:43,012 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:43,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42356 deadline: 1731278443003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:43,012 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:43,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42322 deadline: 1731278443003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:43,012 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:43,012 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:43,013 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:43,013 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:43,013 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:43,013 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:43,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42346 deadline: 1731278443003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:43,013 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:43,013 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:43,013 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:43,013 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:43,191 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:43,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42340 deadline: 1731278443182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:43,192 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 849d0d71310f,43753,1731278250663 for 'test_row_2' in ccb0ca518fd65dd7264624eb4a78050f of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4197 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-10T22:39:43,193 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:43,193 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:43,193 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:43,202 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:43,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42328 deadline: 1731278443193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:43,202 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 849d0d71310f,43753,1731278250663 for 'test_row_1' in ccb0ca518fd65dd7264624eb4a78050f of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4200 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-10T22:39:43,203 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:43,203 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:43,203 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:43,216 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=181 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/A/341c7132a0954a5294c7009c62c4b5f6 2024-11-10T22:39:43,223 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/B/f728c467bea34a3090715001e84b183b is 50, key is test_row_0/B:col10/1731278381645/Put/seqid=0 2024-11-10T22:39:43,227 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:43,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42356 deadline: 1731278443222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:43,228 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:43,228 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:43,228 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:43,228 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:43,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42322 deadline: 1731278443222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:43,228 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:43,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42346 deadline: 1731278443222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:43,228 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:43,228 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:43,228 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:43,229 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:43,229 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:43,229 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:43,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742309_1485 (size=12151) 2024-11-10T22:39:43,542 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:43,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42356 deadline: 1731278443533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:43,542 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:43,543 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:43,543 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:43,543 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:43,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42322 deadline: 1731278443533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:43,543 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:43,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42346 deadline: 1731278443533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:43,543 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:43,543 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:43,543 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:43,543 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:43,543 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:43,543 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:43,631 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=181 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/B/f728c467bea34a3090715001e84b183b 2024-11-10T22:39:43,638 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/C/3a7e8c5f028a401ea14a44aae8911df9 is 50, key is test_row_0/C:col10/1731278381645/Put/seqid=0 2024-11-10T22:39:43,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742310_1486 (size=12151) 2024-11-10T22:39:44,044 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=181 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/C/3a7e8c5f028a401ea14a44aae8911df9 2024-11-10T22:39:44,048 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/A/341c7132a0954a5294c7009c62c4b5f6 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/A/341c7132a0954a5294c7009c62c4b5f6 2024-11-10T22:39:44,051 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/A/341c7132a0954a5294c7009c62c4b5f6, entries=200, sequenceid=181, filesize=14.2 K 2024-11-10T22:39:44,052 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/B/f728c467bea34a3090715001e84b183b as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/B/f728c467bea34a3090715001e84b183b 2024-11-10T22:39:44,055 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/B/f728c467bea34a3090715001e84b183b, entries=150, sequenceid=181, filesize=11.9 K 2024-11-10T22:39:44,056 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/C/3a7e8c5f028a401ea14a44aae8911df9 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/C/3a7e8c5f028a401ea14a44aae8911df9 2024-11-10T22:39:44,059 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/C/3a7e8c5f028a401ea14a44aae8911df9, entries=150, sequenceid=181, filesize=11.9 K 2024-11-10T22:39:44,060 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for ccb0ca518fd65dd7264624eb4a78050f in 1253ms, sequenceid=181, compaction requested=true 2024-11-10T22:39:44,060 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ccb0ca518fd65dd7264624eb4a78050f: 2024-11-10T22:39:44,060 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ccb0ca518fd65dd7264624eb4a78050f:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T22:39:44,060 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T22:39:44,060 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ccb0ca518fd65dd7264624eb4a78050f:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T22:39:44,060 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T22:39:44,060 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T22:39:44,060 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ccb0ca518fd65dd7264624eb4a78050f:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T22:39:44,060 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T22:39:44,060 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T22:39:44,062 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39185 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T22:39:44,062 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T22:39:44,062 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HStore(1541): ccb0ca518fd65dd7264624eb4a78050f/A is initiating minor compaction (all files) 2024-11-10T22:39:44,062 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1541): ccb0ca518fd65dd7264624eb4a78050f/B is initiating minor compaction (all files) 2024-11-10T22:39:44,062 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ccb0ca518fd65dd7264624eb4a78050f/B in TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f. 2024-11-10T22:39:44,062 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ccb0ca518fd65dd7264624eb4a78050f/A in TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f. 2024-11-10T22:39:44,062 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/B/234c945f6b004debad9f25609573ee8a, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/B/ed048410aa224771829b127492153484, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/B/f728c467bea34a3090715001e84b183b] into tmpdir=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp, totalSize=35.9 K 2024-11-10T22:39:44,062 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/A/6e237ee03183468cb7cf9f8d95f49742, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/A/4f39b2de2ad24eec956bce0b5442f63c, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/A/341c7132a0954a5294c7009c62c4b5f6] into tmpdir=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp, totalSize=38.3 K 2024-11-10T22:39:44,063 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting 234c945f6b004debad9f25609573ee8a, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=141, earliestPutTs=1731278378980 2024-11-10T22:39:44,063 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6e237ee03183468cb7cf9f8d95f49742, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=141, earliestPutTs=1731278378980 2024-11-10T22:39:44,063 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting ed048410aa224771829b127492153484, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1731278380438 2024-11-10T22:39:44,063 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4f39b2de2ad24eec956bce0b5442f63c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1731278380438 2024-11-10T22:39:44,063 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting f728c467bea34a3090715001e84b183b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=181, earliestPutTs=1731278381641 2024-11-10T22:39:44,063 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 341c7132a0954a5294c7009c62c4b5f6, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=181, earliestPutTs=1731278381640 2024-11-10T22:39:44,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(8855): Flush requested on ccb0ca518fd65dd7264624eb4a78050f 2024-11-10T22:39:44,064 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T22:39:44,064 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ccb0ca518fd65dd7264624eb4a78050f 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-10T22:39:44,070 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/A/da3cd2f914254827a25cad0585196af7 is 50, key is test_row_0/A:col10/1731278382863/Put/seqid=0 2024-11-10T22:39:44,071 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ccb0ca518fd65dd7264624eb4a78050f#A#compaction#408 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T22:39:44,071 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/A/139d13daf2d54f1f8551a37b92c363c4 is 50, key is test_row_0/A:col10/1731278381645/Put/seqid=0 2024-11-10T22:39:44,073 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ccb0ca518fd65dd7264624eb4a78050f#B#compaction#409 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T22:39:44,074 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/B/2e149c2a7a5c4d4ebd647a115161e201 is 50, key is test_row_0/B:col10/1731278381645/Put/seqid=0 2024-11-10T22:39:44,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742312_1488 (size=12595) 2024-11-10T22:39:44,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742311_1487 (size=16931) 2024-11-10T22:39:44,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742313_1489 (size=12595) 2024-11-10T22:39:44,098 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:44,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42356 deadline: 1731278444086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:44,098 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:44,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42322 deadline: 1731278444087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:44,099 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:44,099 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:44,099 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:44,099 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:44,099 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:44,099 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:44,099 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:44,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42346 deadline: 1731278444089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:44,099 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:44,099 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:44,099 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:44,211 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:44,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42356 deadline: 1731278444202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:44,211 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:44,211 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:44,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42322 deadline: 1731278444202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:44,211 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:44,212 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:44,212 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:44,212 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:44,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42346 deadline: 1731278444202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:44,212 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:44,212 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:44,212 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:44,212 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:44,212 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:44,429 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:44,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42356 deadline: 1731278444422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:44,430 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:44,430 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:44,430 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:44,430 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:44,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42322 deadline: 1731278444422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:44,430 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:44,430 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:44,430 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:44,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42346 deadline: 1731278444422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:44,430 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:44,431 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:44,431 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:44,431 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:44,479 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/A/da3cd2f914254827a25cad0585196af7 2024-11-10T22:39:44,481 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/A/139d13daf2d54f1f8551a37b92c363c4 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/A/139d13daf2d54f1f8551a37b92c363c4 2024-11-10T22:39:44,486 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/B/2e149c2a7a5c4d4ebd647a115161e201 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/B/2e149c2a7a5c4d4ebd647a115161e201 2024-11-10T22:39:44,486 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/B/c0c6e5a65e304bc497a536ee5a1a5cc1 is 50, key is test_row_0/B:col10/1731278382863/Put/seqid=0 2024-11-10T22:39:44,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742314_1490 (size=12151) 2024-11-10T22:39:44,490 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/B/c0c6e5a65e304bc497a536ee5a1a5cc1 2024-11-10T22:39:44,492 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ccb0ca518fd65dd7264624eb4a78050f/A of ccb0ca518fd65dd7264624eb4a78050f into 139d13daf2d54f1f8551a37b92c363c4(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T22:39:44,493 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ccb0ca518fd65dd7264624eb4a78050f: 2024-11-10T22:39:44,493 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., storeName=ccb0ca518fd65dd7264624eb4a78050f/A, priority=13, startTime=1731278384060; duration=0sec 2024-11-10T22:39:44,493 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T22:39:44,493 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ccb0ca518fd65dd7264624eb4a78050f:A 2024-11-10T22:39:44,493 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T22:39:44,493 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ccb0ca518fd65dd7264624eb4a78050f/B of ccb0ca518fd65dd7264624eb4a78050f into 2e149c2a7a5c4d4ebd647a115161e201(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T22:39:44,493 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ccb0ca518fd65dd7264624eb4a78050f: 2024-11-10T22:39:44,493 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., storeName=ccb0ca518fd65dd7264624eb4a78050f/B, priority=13, startTime=1731278384060; duration=0sec 2024-11-10T22:39:44,493 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T22:39:44,493 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ccb0ca518fd65dd7264624eb4a78050f:B 2024-11-10T22:39:44,494 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T22:39:44,494 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HStore(1541): ccb0ca518fd65dd7264624eb4a78050f/C is initiating minor compaction (all files) 2024-11-10T22:39:44,494 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ccb0ca518fd65dd7264624eb4a78050f/C in TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f. 2024-11-10T22:39:44,494 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/C/249507aa08374e0d87ad7b5e55ccaf46, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/C/d02b83d77ad441d9b838760590ef8caa, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/C/3a7e8c5f028a401ea14a44aae8911df9] into tmpdir=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp, totalSize=35.9 K 2024-11-10T22:39:44,495 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 249507aa08374e0d87ad7b5e55ccaf46, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=141, earliestPutTs=1731278378980 2024-11-10T22:39:44,495 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting d02b83d77ad441d9b838760590ef8caa, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1731278380438 2024-11-10T22:39:44,496 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3a7e8c5f028a401ea14a44aae8911df9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=181, earliestPutTs=1731278381641 2024-11-10T22:39:44,498 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/C/c6c25768bb104ff58c6c08c097d4cfa2 is 50, key is test_row_0/C:col10/1731278382863/Put/seqid=0 2024-11-10T22:39:44,502 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ccb0ca518fd65dd7264624eb4a78050f#C#compaction#412 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T22:39:44,502 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/C/4e9b2e5fa195429e9549c94617321513 is 50, key is test_row_0/C:col10/1731278381645/Put/seqid=0 2024-11-10T22:39:44,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742315_1491 (size=12151) 2024-11-10T22:39:44,504 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/C/c6c25768bb104ff58c6c08c097d4cfa2 2024-11-10T22:39:44,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742316_1492 (size=12595) 2024-11-10T22:39:44,507 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/A/da3cd2f914254827a25cad0585196af7 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/A/da3cd2f914254827a25cad0585196af7 2024-11-10T22:39:44,510 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/A/da3cd2f914254827a25cad0585196af7, entries=250, sequenceid=207, filesize=16.5 K 2024-11-10T22:39:44,511 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/B/c0c6e5a65e304bc497a536ee5a1a5cc1 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/B/c0c6e5a65e304bc497a536ee5a1a5cc1 2024-11-10T22:39:44,518 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/B/c0c6e5a65e304bc497a536ee5a1a5cc1, entries=150, sequenceid=207, filesize=11.9 K 2024-11-10T22:39:44,519 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/C/c6c25768bb104ff58c6c08c097d4cfa2 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/C/c6c25768bb104ff58c6c08c097d4cfa2 2024-11-10T22:39:44,522 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/C/c6c25768bb104ff58c6c08c097d4cfa2, entries=150, sequenceid=207, filesize=11.9 K 2024-11-10T22:39:44,523 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for ccb0ca518fd65dd7264624eb4a78050f in 459ms, sequenceid=207, compaction requested=false 2024-11-10T22:39:44,523 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ccb0ca518fd65dd7264624eb4a78050f: 2024-11-10T22:39:44,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-10T22:39:44,563 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T22:39:44,563 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T22:39:44,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] procedure2.ProcedureExecutor(1139): Stored pid=137, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees 2024-11-10T22:39:44,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-10T22:39:44,565 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=137, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T22:39:44,566 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=137, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T22:39:44,566 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T22:39:44,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-10T22:39:44,718 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-10T22:39:44,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f. 2024-11-10T22:39:44,719 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2902): Flushing ccb0ca518fd65dd7264624eb4a78050f 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-10T22:39:44,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/A/3677e016e8864fa2ad2b7054e695491a is 50, key is test_row_0/A:col10/1731278384086/Put/seqid=0 2024-11-10T22:39:44,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742317_1493 (size=12151) 2024-11-10T22:39:44,727 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=219 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/A/3677e016e8864fa2ad2b7054e695491a 2024-11-10T22:39:44,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/B/e07eaacdac0b43309977ecb4d30ae2cb is 50, key is test_row_0/B:col10/1731278384086/Put/seqid=0 2024-11-10T22:39:44,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(8855): Flush requested on ccb0ca518fd65dd7264624eb4a78050f 2024-11-10T22:39:44,737 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f. as already flushing 2024-11-10T22:39:44,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742318_1494 (size=12151) 2024-11-10T22:39:44,825 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:44,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42322 deadline: 1731278444814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:44,825 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:44,826 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:44,826 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:44,826 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:44,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42346 deadline: 1731278444818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:44,827 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:44,827 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:44,827 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:44,837 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:44,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42356 deadline: 1731278444820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:44,838 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:44,838 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:44,838 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:44,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-10T22:39:44,911 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/C/4e9b2e5fa195429e9549c94617321513 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/C/4e9b2e5fa195429e9549c94617321513 2024-11-10T22:39:44,914 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ccb0ca518fd65dd7264624eb4a78050f/C of ccb0ca518fd65dd7264624eb4a78050f into 4e9b2e5fa195429e9549c94617321513(size=12.3 K), total size for store is 24.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T22:39:44,914 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ccb0ca518fd65dd7264624eb4a78050f: 2024-11-10T22:39:44,914 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., storeName=ccb0ca518fd65dd7264624eb4a78050f/C, priority=13, startTime=1731278384060; duration=0sec 2024-11-10T22:39:44,915 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T22:39:44,915 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ccb0ca518fd65dd7264624eb4a78050f:C 2024-11-10T22:39:44,942 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:44,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42322 deadline: 1731278444932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:44,942 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:44,942 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:44,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42346 deadline: 1731278444932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:44,943 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:44,943 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:44,943 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:44,943 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:44,943 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:44,947 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:44,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42356 deadline: 1731278444943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:44,947 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:44,948 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:44,948 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:45,139 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=219 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/B/e07eaacdac0b43309977ecb4d30ae2cb 2024-11-10T22:39:45,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/C/0dc7102c2e914874808b456945580fdb is 50, key is test_row_0/C:col10/1731278384086/Put/seqid=0 2024-11-10T22:39:45,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742319_1495 (size=12151) 2024-11-10T22:39:45,155 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:45,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42322 deadline: 1731278445152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:45,156 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:45,156 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:45,156 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:45,156 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:45,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42346 deadline: 1731278445152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:45,156 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:45,156 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:45,156 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:45,156 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:45,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42356 deadline: 1731278445152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:45,156 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:45,157 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:45,157 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:45,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-10T22:39:45,467 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:45,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42322 deadline: 1731278445462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:45,467 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:45,468 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:45,468 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:45,472 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:45,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42346 deadline: 1731278445462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:45,472 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:45,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42356 deadline: 1731278445462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:45,472 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:45,473 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:45,473 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:45,473 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:45,473 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:45,473 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:45,550 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=219 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/C/0dc7102c2e914874808b456945580fdb 2024-11-10T22:39:45,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/A/3677e016e8864fa2ad2b7054e695491a as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/A/3677e016e8864fa2ad2b7054e695491a 2024-11-10T22:39:45,557 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/A/3677e016e8864fa2ad2b7054e695491a, entries=150, sequenceid=219, filesize=11.9 K 2024-11-10T22:39:45,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/B/e07eaacdac0b43309977ecb4d30ae2cb as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/B/e07eaacdac0b43309977ecb4d30ae2cb 2024-11-10T22:39:45,560 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/B/e07eaacdac0b43309977ecb4d30ae2cb, entries=150, sequenceid=219, filesize=11.9 K 2024-11-10T22:39:45,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/C/0dc7102c2e914874808b456945580fdb as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/C/0dc7102c2e914874808b456945580fdb 2024-11-10T22:39:45,565 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/C/0dc7102c2e914874808b456945580fdb, entries=150, sequenceid=219, filesize=11.9 K 2024-11-10T22:39:45,566 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=154.31 KB/158010 for ccb0ca518fd65dd7264624eb4a78050f in 846ms, sequenceid=219, compaction requested=true 2024-11-10T22:39:45,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2603): Flush status journal for ccb0ca518fd65dd7264624eb4a78050f: 2024-11-10T22:39:45,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f. 2024-11-10T22:39:45,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=138 2024-11-10T22:39:45,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.HMaster(4169): Remote procedure done, pid=138 2024-11-10T22:39:45,569 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-11-10T22:39:45,569 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0010 sec 2024-11-10T22:39:45,571 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees in 1.0070 sec 2024-11-10T22:39:45,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-10T22:39:45,702 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T22:39:45,703 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T22:39:45,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] procedure2.ProcedureExecutor(1139): Stored pid=139, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees 2024-11-10T22:39:45,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-10T22:39:45,704 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=139, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T22:39:45,705 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=139, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T22:39:45,705 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T22:39:45,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-10T22:39:45,857 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-10T22:39:45,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f. 2024-11-10T22:39:45,858 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2902): Flushing ccb0ca518fd65dd7264624eb4a78050f 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-10T22:39:45,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/A/91ebb218427b4926a65143dad7f2285c is 50, key is test_row_0/A:col10/1731278384811/Put/seqid=0 2024-11-10T22:39:45,867 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742320_1496 (size=12151) 2024-11-10T22:39:45,868 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=246 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/A/91ebb218427b4926a65143dad7f2285c 2024-11-10T22:39:45,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/B/84e06cbedab5468db22af82a2e00c4ff is 50, key is test_row_0/B:col10/1731278384811/Put/seqid=0 2024-11-10T22:39:45,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742321_1497 (size=12151) 2024-11-10T22:39:45,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(8855): Flush requested on ccb0ca518fd65dd7264624eb4a78050f 2024-11-10T22:39:45,977 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f. as already flushing 2024-11-10T22:39:46,000 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:46,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42322 deadline: 1731278445990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:46,001 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:46,001 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:46,001 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:46,002 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:46,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42346 deadline: 1731278446000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:46,003 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:46,003 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:46,003 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:46,013 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:46,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42356 deadline: 1731278446001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:46,014 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:46,014 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:46,014 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:46,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-10T22:39:46,124 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:46,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42322 deadline: 1731278446113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:46,124 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:46,124 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:46,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42346 deadline: 1731278446113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:46,124 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:46,124 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:46,124 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:46,124 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:46,124 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:46,133 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:46,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42356 deadline: 1731278446122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:46,133 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:46,133 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:46,133 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:46,279 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=246 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/B/84e06cbedab5468db22af82a2e00c4ff 2024-11-10T22:39:46,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/C/b8ee7fc7e2f94ae2bc4dbad6a13774be is 50, key is test_row_0/C:col10/1731278384811/Put/seqid=0 2024-11-10T22:39:46,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742322_1498 (size=12151) 2024-11-10T22:39:46,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-10T22:39:46,339 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:46,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42322 deadline: 1731278446333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:46,340 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:46,340 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:46,340 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:46,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42346 deadline: 1731278446333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:46,340 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:46,340 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:46,340 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:46,340 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:46,356 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:46,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42356 deadline: 1731278446342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:46,356 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:46,356 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:46,356 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:46,653 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:46,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42322 deadline: 1731278446642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:46,654 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:46,654 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:46,654 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:46,663 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:46,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42346 deadline: 1731278446652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:46,664 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:46,664 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:46,664 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:46,671 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:46,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42356 deadline: 1731278446663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:46,672 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:46,672 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:46,672 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:46,690 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=246 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/C/b8ee7fc7e2f94ae2bc4dbad6a13774be 2024-11-10T22:39:46,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/A/91ebb218427b4926a65143dad7f2285c as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/A/91ebb218427b4926a65143dad7f2285c 2024-11-10T22:39:46,698 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/A/91ebb218427b4926a65143dad7f2285c, entries=150, sequenceid=246, filesize=11.9 K 2024-11-10T22:39:46,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/B/84e06cbedab5468db22af82a2e00c4ff as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/B/84e06cbedab5468db22af82a2e00c4ff 2024-11-10T22:39:46,702 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/B/84e06cbedab5468db22af82a2e00c4ff, entries=150, sequenceid=246, filesize=11.9 K 2024-11-10T22:39:46,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/C/b8ee7fc7e2f94ae2bc4dbad6a13774be as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/C/b8ee7fc7e2f94ae2bc4dbad6a13774be 2024-11-10T22:39:46,706 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/C/b8ee7fc7e2f94ae2bc4dbad6a13774be, entries=150, sequenceid=246, filesize=11.9 K 2024-11-10T22:39:46,706 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for ccb0ca518fd65dd7264624eb4a78050f in 849ms, sequenceid=246, compaction requested=true 2024-11-10T22:39:46,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2603): Flush status journal for ccb0ca518fd65dd7264624eb4a78050f: 2024-11-10T22:39:46,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f. 2024-11-10T22:39:46,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=140 2024-11-10T22:39:46,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.HMaster(4169): Remote procedure done, pid=140 2024-11-10T22:39:46,710 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=140, resume processing ppid=139 2024-11-10T22:39:46,710 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, ppid=139, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0030 sec 2024-11-10T22:39:46,713 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees in 1.0080 sec 2024-11-10T22:39:46,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-10T22:39:46,842 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T22:39:46,843 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T22:39:46,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] procedure2.ProcedureExecutor(1139): Stored pid=141, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=141, table=TestAcidGuarantees 2024-11-10T22:39:46,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-10T22:39:46,845 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=141, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=141, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T22:39:46,846 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=141, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=141, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T22:39:46,846 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=142, ppid=141, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T22:39:46,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-10T22:39:46,998 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-10T22:39:46,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f. 2024-11-10T22:39:46,998 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2902): Flushing ccb0ca518fd65dd7264624eb4a78050f 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-10T22:39:47,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/A/2fd383d277644494ba599019cf02b141 is 50, key is test_row_0/A:col10/1731278385998/Put/seqid=0 2024-11-10T22:39:47,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742323_1499 (size=12151) 2024-11-10T22:39:47,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-10T22:39:47,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(8855): Flush requested on ccb0ca518fd65dd7264624eb4a78050f 2024-11-10T22:39:47,169 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f. as already flushing 2024-11-10T22:39:47,278 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:47,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42328 deadline: 1731278447273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:47,278 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:47,278 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:47,278 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:47,285 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:47,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42356 deadline: 1731278447275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:47,286 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:47,286 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:47,286 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:47,292 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:47,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42340 deadline: 1731278447276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:47,292 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:47,292 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:47,292 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:47,292 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:47,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42346 deadline: 1731278447277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:47,292 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:47,292 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:47,292 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:47,292 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:47,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42322 deadline: 1731278447278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:47,292 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:47,293 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:47,293 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:47,393 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:47,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42328 deadline: 1731278447382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:47,394 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:47,394 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:47,394 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:47,399 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:47,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42356 deadline: 1731278447392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:47,400 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:47,400 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:47,400 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:47,408 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=256 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/A/2fd383d277644494ba599019cf02b141 2024-11-10T22:39:47,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/B/e1a484f4837c446cbc1e49a012727be8 is 50, key is test_row_0/B:col10/1731278385998/Put/seqid=0 2024-11-10T22:39:47,415 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:47,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42340 deadline: 1731278447402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:47,415 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:47,415 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:47,415 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:47,415 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:47,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42346 deadline: 1731278447403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:47,415 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:47,415 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:47,415 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:47,415 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:47,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42322 deadline: 1731278447403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:47,416 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:47,416 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:47,416 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:47,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742324_1500 (size=12151) 2024-11-10T22:39:47,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-10T22:39:47,611 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:47,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42328 deadline: 1731278447602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:47,612 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:47,612 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:47,612 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:47,616 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:47,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42356 deadline: 1731278447613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:47,616 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:47,616 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:47,616 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:47,631 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:47,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42340 deadline: 1731278447622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:47,631 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:47,631 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:47,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42346 deadline: 1731278447622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:47,631 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:47,631 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:47,631 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:47,631 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:47,632 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:47,632 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:47,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42322 deadline: 1731278447622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:47,632 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:47,632 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:47,632 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:47,819 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=256 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/B/e1a484f4837c446cbc1e49a012727be8 2024-11-10T22:39:47,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/C/62d497f73010486893406b9d9d94b349 is 50, key is test_row_0/C:col10/1731278385998/Put/seqid=0 2024-11-10T22:39:47,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742325_1501 (size=12151) 2024-11-10T22:39:47,931 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:47,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42328 deadline: 1731278447922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:47,932 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:47,932 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:47,932 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:47,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42356 deadline: 1731278447923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:47,932 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:47,932 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:47,932 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:47,932 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:47,951 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:47,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42340 deadline: 1731278447942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:47,951 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:47,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42346 deadline: 1731278447942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:47,952 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:47,952 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:47,952 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:47,952 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:47,952 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:47,952 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:47,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42322 deadline: 1731278447942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:47,952 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:47,952 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:47,952 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:47,952 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:47,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-10T22:39:48,232 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=256 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/C/62d497f73010486893406b9d9d94b349 2024-11-10T22:39:48,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/A/2fd383d277644494ba599019cf02b141 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/A/2fd383d277644494ba599019cf02b141 2024-11-10T22:39:48,239 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/A/2fd383d277644494ba599019cf02b141, entries=150, sequenceid=256, filesize=11.9 K 2024-11-10T22:39:48,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/B/e1a484f4837c446cbc1e49a012727be8 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/B/e1a484f4837c446cbc1e49a012727be8 2024-11-10T22:39:48,242 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/B/e1a484f4837c446cbc1e49a012727be8, entries=150, sequenceid=256, filesize=11.9 K 2024-11-10T22:39:48,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/C/62d497f73010486893406b9d9d94b349 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/C/62d497f73010486893406b9d9d94b349 2024-11-10T22:39:48,246 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/C/62d497f73010486893406b9d9d94b349, entries=150, sequenceid=256, filesize=11.9 K 2024-11-10T22:39:48,247 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=154.31 KB/158010 for ccb0ca518fd65dd7264624eb4a78050f in 1248ms, sequenceid=256, compaction requested=true 2024-11-10T22:39:48,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2603): Flush status journal for ccb0ca518fd65dd7264624eb4a78050f: 2024-11-10T22:39:48,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f. 2024-11-10T22:39:48,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=142 2024-11-10T22:39:48,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.HMaster(4169): Remote procedure done, pid=142 2024-11-10T22:39:48,250 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=142, resume processing ppid=141 2024-11-10T22:39:48,250 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, ppid=141, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4020 sec 2024-11-10T22:39:48,251 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=141, table=TestAcidGuarantees in 1.4070 sec 2024-11-10T22:39:48,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(8855): Flush requested on ccb0ca518fd65dd7264624eb4a78050f 2024-11-10T22:39:48,450 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T22:39:48,450 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ccb0ca518fd65dd7264624eb4a78050f 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-10T22:39:48,455 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/A/aef42e86ee384591a0a6939fe3226347 is 50, key is test_row_0/A:col10/1731278388448/Put/seqid=0 2024-11-10T22:39:48,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742326_1502 (size=17181) 2024-11-10T22:39:48,474 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:48,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42356 deadline: 1731278448463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:48,474 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:48,474 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:48,474 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:48,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42328 deadline: 1731278448466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:48,474 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:48,474 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:48,475 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:48,475 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:48,477 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:48,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42340 deadline: 1731278448473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:48,477 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:48,477 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:48,477 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:48,486 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:48,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42346 deadline: 1731278448474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:48,486 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:48,487 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:48,487 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:48,486 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:48,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42322 deadline: 1731278448474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:48,487 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:48,487 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:48,487 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:48,590 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:48,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42356 deadline: 1731278448583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:48,590 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:48,590 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:48,590 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:48,590 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:48,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42328 deadline: 1731278448583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:48,591 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:48,591 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:48,591 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:48,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42340 deadline: 1731278448583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:48,591 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:48,591 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:48,591 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:48,591 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:48,603 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:48,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42346 deadline: 1731278448592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:48,604 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:48,604 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:48,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42322 deadline: 1731278448592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:48,604 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:48,604 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:48,604 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:48,604 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:48,604 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:48,803 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:48,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42356 deadline: 1731278448793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:48,803 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:48,803 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:48,803 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:48,803 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:48,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42340 deadline: 1731278448793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:48,804 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:48,804 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:48,804 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:48,810 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:48,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42328 deadline: 1731278448802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:48,811 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:48,811 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:48,811 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:48,819 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:48,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42346 deadline: 1731278448812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:48,819 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:48,819 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:48,820 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:48,824 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:48,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42322 deadline: 1731278448812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:48,824 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:48,824 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:48,824 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:48,859 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=284 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/A/aef42e86ee384591a0a6939fe3226347 2024-11-10T22:39:48,866 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/B/4455656a67eb430e898492f51897ec9b is 50, key is test_row_0/B:col10/1731278388448/Put/seqid=0 2024-11-10T22:39:48,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742327_1503 (size=12301) 2024-11-10T22:39:48,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-10T22:39:48,993 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T22:39:48,993 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T22:39:48,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] procedure2.ProcedureExecutor(1139): Stored pid=143, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=143, table=TestAcidGuarantees 2024-11-10T22:39:48,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-10T22:39:48,995 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=143, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=143, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T22:39:48,996 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=143, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=143, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T22:39:48,996 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=144, ppid=143, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T22:39:49,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-10T22:39:49,116 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:49,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42356 deadline: 1731278449112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:49,117 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:49,117 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:49,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42340 deadline: 1731278449112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:49,117 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:49,117 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:49,117 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:49,117 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:49,117 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:49,130 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:49,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42328 deadline: 1731278449122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:49,131 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:49,131 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:49,131 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:49,144 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:49,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42346 deadline: 1731278449132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:49,144 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:49,144 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:49,144 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:49,144 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:49,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42322 deadline: 1731278449133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:49,145 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:49,145 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:49,145 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:49,148 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=144 2024-11-10T22:39:49,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f. 2024-11-10T22:39:49,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f. as already flushing 2024-11-10T22:39:49,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f. 2024-11-10T22:39:49,148 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] handler.RSProcedureHandler(58): pid=144 java.io.IOException: Unable to complete flush {ENCODED => ccb0ca518fd65dd7264624eb4a78050f, NAME => 'TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:49,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=144 java.io.IOException: Unable to complete flush {ENCODED => ccb0ca518fd65dd7264624eb4a78050f, NAME => 'TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:49,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=144 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ccb0ca518fd65dd7264624eb4a78050f, NAME => 'TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ccb0ca518fd65dd7264624eb4a78050f, NAME => 'TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:49,270 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=284 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/B/4455656a67eb430e898492f51897ec9b 2024-11-10T22:39:49,279 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/C/fbe195f9ef534ffa9e1526e73df2edd4 is 50, key is test_row_0/C:col10/1731278388448/Put/seqid=0 2024-11-10T22:39:49,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742328_1504 (size=12301) 2024-11-10T22:39:49,301 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=144 2024-11-10T22:39:49,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f. 2024-11-10T22:39:49,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f. as already flushing 2024-11-10T22:39:49,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f. 2024-11-10T22:39:49,301 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] handler.RSProcedureHandler(58): pid=144 java.io.IOException: Unable to complete flush {ENCODED => ccb0ca518fd65dd7264624eb4a78050f, NAME => 'TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:49,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=144 java.io.IOException: Unable to complete flush {ENCODED => ccb0ca518fd65dd7264624eb4a78050f, NAME => 'TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:49,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=144 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ccb0ca518fd65dd7264624eb4a78050f, NAME => 'TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ccb0ca518fd65dd7264624eb4a78050f, NAME => 'TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:49,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-10T22:39:49,453 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=144 2024-11-10T22:39:49,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f. 2024-11-10T22:39:49,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f. as already flushing 2024-11-10T22:39:49,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f. 2024-11-10T22:39:49,454 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] handler.RSProcedureHandler(58): pid=144 java.io.IOException: Unable to complete flush {ENCODED => ccb0ca518fd65dd7264624eb4a78050f, NAME => 'TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:49,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=144 java.io.IOException: Unable to complete flush {ENCODED => ccb0ca518fd65dd7264624eb4a78050f, NAME => 'TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:49,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=144 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ccb0ca518fd65dd7264624eb4a78050f, NAME => 'TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ccb0ca518fd65dd7264624eb4a78050f, NAME => 'TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:49,606 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=144 2024-11-10T22:39:49,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f. 2024-11-10T22:39:49,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f. as already flushing 2024-11-10T22:39:49,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f. 2024-11-10T22:39:49,607 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] handler.RSProcedureHandler(58): pid=144 java.io.IOException: Unable to complete flush {ENCODED => ccb0ca518fd65dd7264624eb4a78050f, NAME => 'TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:49,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=144 java.io.IOException: Unable to complete flush {ENCODED => ccb0ca518fd65dd7264624eb4a78050f, NAME => 'TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:49,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=144 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ccb0ca518fd65dd7264624eb4a78050f, NAME => 'TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ccb0ca518fd65dd7264624eb4a78050f, NAME => 'TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:49,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-10T22:39:49,631 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:49,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42356 deadline: 1731278449622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:49,631 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:49,632 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:49,632 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:49,642 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:49,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42340 deadline: 1731278449633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:49,642 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:49,642 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:49,642 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:49,649 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:49,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42328 deadline: 1731278449642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:49,650 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:49,650 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:49,650 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:49,665 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:49,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42346 deadline: 1731278449652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:49,665 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:49,665 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:49,665 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:49,665 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:49,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42322 deadline: 1731278449652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:49,666 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:49,666 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:49,666 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:49,683 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=284 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/C/fbe195f9ef534ffa9e1526e73df2edd4 2024-11-10T22:39:49,687 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/A/aef42e86ee384591a0a6939fe3226347 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/A/aef42e86ee384591a0a6939fe3226347 2024-11-10T22:39:49,691 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/A/aef42e86ee384591a0a6939fe3226347, entries=250, sequenceid=284, filesize=16.8 K 2024-11-10T22:39:49,691 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/B/4455656a67eb430e898492f51897ec9b as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/B/4455656a67eb430e898492f51897ec9b 2024-11-10T22:39:49,695 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/B/4455656a67eb430e898492f51897ec9b, entries=150, sequenceid=284, filesize=12.0 K 2024-11-10T22:39:49,696 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/C/fbe195f9ef534ffa9e1526e73df2edd4 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/C/fbe195f9ef534ffa9e1526e73df2edd4 2024-11-10T22:39:49,699 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/C/fbe195f9ef534ffa9e1526e73df2edd4, entries=150, sequenceid=284, filesize=12.0 K 2024-11-10T22:39:49,699 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=40.25 KB/41220 for ccb0ca518fd65dd7264624eb4a78050f in 1249ms, sequenceid=284, compaction requested=true 2024-11-10T22:39:49,699 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ccb0ca518fd65dd7264624eb4a78050f: 2024-11-10T22:39:49,700 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ccb0ca518fd65dd7264624eb4a78050f:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T22:39:49,700 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T22:39:49,700 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ccb0ca518fd65dd7264624eb4a78050f:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T22:39:49,700 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T22:39:49,700 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-11-10T22:39:49,700 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ccb0ca518fd65dd7264624eb4a78050f:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T22:39:49,700 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-11-10T22:39:49,700 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T22:39:49,701 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 73500 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-11-10T22:39:49,701 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 83160 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-11-10T22:39:49,701 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1541): ccb0ca518fd65dd7264624eb4a78050f/B is initiating minor compaction (all files) 2024-11-10T22:39:49,701 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HStore(1541): ccb0ca518fd65dd7264624eb4a78050f/A is initiating minor compaction (all files) 2024-11-10T22:39:49,701 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ccb0ca518fd65dd7264624eb4a78050f/B in TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f. 2024-11-10T22:39:49,701 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ccb0ca518fd65dd7264624eb4a78050f/A in TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f. 2024-11-10T22:39:49,701 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/B/2e149c2a7a5c4d4ebd647a115161e201, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/B/c0c6e5a65e304bc497a536ee5a1a5cc1, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/B/e07eaacdac0b43309977ecb4d30ae2cb, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/B/84e06cbedab5468db22af82a2e00c4ff, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/B/e1a484f4837c446cbc1e49a012727be8, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/B/4455656a67eb430e898492f51897ec9b] into tmpdir=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp, totalSize=71.8 K 2024-11-10T22:39:49,701 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/A/139d13daf2d54f1f8551a37b92c363c4, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/A/da3cd2f914254827a25cad0585196af7, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/A/3677e016e8864fa2ad2b7054e695491a, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/A/91ebb218427b4926a65143dad7f2285c, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/A/2fd383d277644494ba599019cf02b141, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/A/aef42e86ee384591a0a6939fe3226347] into tmpdir=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp, totalSize=81.2 K 2024-11-10T22:39:49,702 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 139d13daf2d54f1f8551a37b92c363c4, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=181, earliestPutTs=1731278381641 2024-11-10T22:39:49,702 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting da3cd2f914254827a25cad0585196af7, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1731278382861 2024-11-10T22:39:49,702 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting 2e149c2a7a5c4d4ebd647a115161e201, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=181, earliestPutTs=1731278381641 2024-11-10T22:39:49,702 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3677e016e8864fa2ad2b7054e695491a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=219, earliestPutTs=1731278384084 2024-11-10T22:39:49,703 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting c0c6e5a65e304bc497a536ee5a1a5cc1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1731278382863 2024-11-10T22:39:49,703 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 91ebb218427b4926a65143dad7f2285c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1731278384811 2024-11-10T22:39:49,703 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2fd383d277644494ba599019cf02b141, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1731278385989 2024-11-10T22:39:49,703 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting e07eaacdac0b43309977ecb4d30ae2cb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=219, earliestPutTs=1731278384084 2024-11-10T22:39:49,703 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting aef42e86ee384591a0a6939fe3226347, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1731278387274 2024-11-10T22:39:49,703 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting 84e06cbedab5468db22af82a2e00c4ff, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1731278384811 2024-11-10T22:39:49,703 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting e1a484f4837c446cbc1e49a012727be8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1731278385989 2024-11-10T22:39:49,704 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting 4455656a67eb430e898492f51897ec9b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1731278387275 2024-11-10T22:39:49,714 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ccb0ca518fd65dd7264624eb4a78050f#B#compaction#426 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T22:39:49,715 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ccb0ca518fd65dd7264624eb4a78050f#A#compaction#425 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T22:39:49,715 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/B/c2321e6fbcdf40c4851ccfac6223a4ef is 50, key is test_row_0/B:col10/1731278388448/Put/seqid=0 2024-11-10T22:39:49,715 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/A/7edfff33a077497a83e18e3f9a39d5af is 50, key is test_row_0/A:col10/1731278388448/Put/seqid=0 2024-11-10T22:39:49,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742329_1505 (size=12949) 2024-11-10T22:39:49,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742330_1506 (size=12949) 2024-11-10T22:39:49,760 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=144 2024-11-10T22:39:49,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f. 2024-11-10T22:39:49,760 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegion(2902): Flushing ccb0ca518fd65dd7264624eb4a78050f 3/3 column families, dataSize=40.25 KB heapSize=106.22 KB 2024-11-10T22:39:49,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/A/15d1df3cb18b4b029f7f88f49c3e71d8 is 50, key is test_row_0/A:col10/1731278388473/Put/seqid=0 2024-11-10T22:39:49,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742331_1507 (size=12301) 2024-11-10T22:39:50,126 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/B/c2321e6fbcdf40c4851ccfac6223a4ef as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/B/c2321e6fbcdf40c4851ccfac6223a4ef 2024-11-10T22:39:50,127 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/A/7edfff33a077497a83e18e3f9a39d5af as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/A/7edfff33a077497a83e18e3f9a39d5af 2024-11-10T22:39:50,131 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in ccb0ca518fd65dd7264624eb4a78050f/A of ccb0ca518fd65dd7264624eb4a78050f into 7edfff33a077497a83e18e3f9a39d5af(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T22:39:50,131 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in ccb0ca518fd65dd7264624eb4a78050f/B of ccb0ca518fd65dd7264624eb4a78050f into c2321e6fbcdf40c4851ccfac6223a4ef(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T22:39:50,131 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ccb0ca518fd65dd7264624eb4a78050f: 2024-11-10T22:39:50,131 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ccb0ca518fd65dd7264624eb4a78050f: 2024-11-10T22:39:50,131 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., storeName=ccb0ca518fd65dd7264624eb4a78050f/B, priority=10, startTime=1731278389700; duration=0sec 2024-11-10T22:39:50,131 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., storeName=ccb0ca518fd65dd7264624eb4a78050f/A, priority=10, startTime=1731278389700; duration=0sec 2024-11-10T22:39:50,131 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T22:39:50,131 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ccb0ca518fd65dd7264624eb4a78050f:B 2024-11-10T22:39:50,131 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-11-10T22:39:50,131 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T22:39:50,131 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ccb0ca518fd65dd7264624eb4a78050f:A 2024-11-10T22:39:50,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-10T22:39:50,134 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 73500 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-11-10T22:39:50,134 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1541): ccb0ca518fd65dd7264624eb4a78050f/C is initiating minor compaction (all files) 2024-11-10T22:39:50,134 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ccb0ca518fd65dd7264624eb4a78050f/C in TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f. 2024-11-10T22:39:50,134 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/C/4e9b2e5fa195429e9549c94617321513, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/C/c6c25768bb104ff58c6c08c097d4cfa2, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/C/0dc7102c2e914874808b456945580fdb, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/C/b8ee7fc7e2f94ae2bc4dbad6a13774be, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/C/62d497f73010486893406b9d9d94b349, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/C/fbe195f9ef534ffa9e1526e73df2edd4] into tmpdir=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp, totalSize=71.8 K 2024-11-10T22:39:50,135 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting 4e9b2e5fa195429e9549c94617321513, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=181, earliestPutTs=1731278381641 2024-11-10T22:39:50,135 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting c6c25768bb104ff58c6c08c097d4cfa2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1731278382863 2024-11-10T22:39:50,135 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting 0dc7102c2e914874808b456945580fdb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=219, earliestPutTs=1731278384084 2024-11-10T22:39:50,135 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting b8ee7fc7e2f94ae2bc4dbad6a13774be, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1731278384811 2024-11-10T22:39:50,136 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting 62d497f73010486893406b9d9d94b349, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1731278385989 2024-11-10T22:39:50,136 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting fbe195f9ef534ffa9e1526e73df2edd4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1731278387275 2024-11-10T22:39:50,152 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ccb0ca518fd65dd7264624eb4a78050f#C#compaction#428 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T22:39:50,152 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/C/d03a6f51ae9d489f887275fdd4ee42df is 50, key is test_row_0/C:col10/1731278388448/Put/seqid=0 2024-11-10T22:39:50,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742332_1508 (size=12949) 2024-11-10T22:39:50,165 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/C/d03a6f51ae9d489f887275fdd4ee42df as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/C/d03a6f51ae9d489f887275fdd4ee42df 2024-11-10T22:39:50,169 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/A/15d1df3cb18b4b029f7f88f49c3e71d8 2024-11-10T22:39:50,173 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in ccb0ca518fd65dd7264624eb4a78050f/C of ccb0ca518fd65dd7264624eb4a78050f into d03a6f51ae9d489f887275fdd4ee42df(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T22:39:50,173 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ccb0ca518fd65dd7264624eb4a78050f: 2024-11-10T22:39:50,173 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., storeName=ccb0ca518fd65dd7264624eb4a78050f/C, priority=10, startTime=1731278389700; duration=0sec 2024-11-10T22:39:50,173 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T22:39:50,173 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ccb0ca518fd65dd7264624eb4a78050f:C 2024-11-10T22:39:50,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/B/0619c07356fd497d8e316071028ebb56 is 50, key is test_row_0/B:col10/1731278388473/Put/seqid=0 2024-11-10T22:39:50,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742333_1509 (size=12301) 2024-11-10T22:39:50,583 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/B/0619c07356fd497d8e316071028ebb56 2024-11-10T22:39:50,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/C/3f207e2fe78e4932906fcea2672cdd7c is 50, key is test_row_0/C:col10/1731278388473/Put/seqid=0 2024-11-10T22:39:50,596 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742334_1510 (size=12301) 2024-11-10T22:39:50,597 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/C/3f207e2fe78e4932906fcea2672cdd7c 2024-11-10T22:39:50,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/A/15d1df3cb18b4b029f7f88f49c3e71d8 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/A/15d1df3cb18b4b029f7f88f49c3e71d8 2024-11-10T22:39:50,606 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/A/15d1df3cb18b4b029f7f88f49c3e71d8, entries=150, sequenceid=293, filesize=12.0 K 2024-11-10T22:39:50,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/B/0619c07356fd497d8e316071028ebb56 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/B/0619c07356fd497d8e316071028ebb56 2024-11-10T22:39:50,613 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/B/0619c07356fd497d8e316071028ebb56, entries=150, sequenceid=293, filesize=12.0 K 2024-11-10T22:39:50,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/C/3f207e2fe78e4932906fcea2672cdd7c as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/C/3f207e2fe78e4932906fcea2672cdd7c 2024-11-10T22:39:50,617 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/C/3f207e2fe78e4932906fcea2672cdd7c, entries=150, sequenceid=293, filesize=12.0 K 2024-11-10T22:39:50,619 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.17 KB/108720, currentSize=0 B/0 for ccb0ca518fd65dd7264624eb4a78050f in 858ms, sequenceid=293, compaction requested=false 2024-11-10T22:39:50,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegion(2603): Flush status journal for ccb0ca518fd65dd7264624eb4a78050f: 2024-11-10T22:39:50,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f. 2024-11-10T22:39:50,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=144 2024-11-10T22:39:50,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.HMaster(4169): Remote procedure done, pid=144 2024-11-10T22:39:50,624 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=144, resume processing ppid=143 2024-11-10T22:39:50,624 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, ppid=143, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6240 sec 2024-11-10T22:39:50,628 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=143, table=TestAcidGuarantees in 1.6310 sec 2024-11-10T22:39:50,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(8855): Flush requested on ccb0ca518fd65dd7264624eb4a78050f 2024-11-10T22:39:50,664 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T22:39:50,664 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ccb0ca518fd65dd7264624eb4a78050f 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-10T22:39:50,670 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/A/2784522906f2418c9e206c9218f6afa7 is 50, key is test_row_0/A:col10/1731278390655/Put/seqid=0 2024-11-10T22:39:50,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742335_1511 (size=14741) 2024-11-10T22:39:50,731 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:50,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42356 deadline: 1731278450718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:50,732 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:50,732 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:50,732 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:50,743 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:50,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42346 deadline: 1731278450726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:50,744 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:50,744 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:50,744 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:50,744 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:50,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42328 deadline: 1731278450727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:50,744 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:50,744 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:50,744 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:50,744 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:50,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42340 deadline: 1731278450729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:50,744 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:50,744 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:50,744 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:50,744 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:50,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42322 deadline: 1731278450730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:50,745 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:50,745 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:50,745 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:50,852 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:50,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42356 deadline: 1731278450842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:50,853 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:50,853 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:50,853 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:50,856 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:50,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42346 deadline: 1731278450852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:50,857 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:50,857 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:50,857 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:50,857 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:50,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42328 deadline: 1731278450852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:50,857 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:50,857 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:50,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42340 deadline: 1731278450852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:50,857 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:50,857 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:50,857 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:50,857 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:50,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42322 deadline: 1731278450852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:50,857 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:50,857 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:50,858 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:50,858 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:50,858 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:51,071 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:51,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42356 deadline: 1731278451062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:51,071 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:51,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42346 deadline: 1731278451062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:51,071 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:51,071 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:51,071 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:51,071 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:51,071 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:51,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42328 deadline: 1731278451062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:51,071 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:51,071 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:51,072 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:51,072 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:51,072 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:51,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42340 deadline: 1731278451063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:51,072 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:51,072 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:51,072 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:51,072 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:51,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42322 deadline: 1731278451063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:51,072 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:51,072 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:51,072 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:51,072 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:51,077 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=307 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/A/2784522906f2418c9e206c9218f6afa7 2024-11-10T22:39:51,084 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/B/6d60846d3c2c41899e7831d1dc4528c6 is 50, key is test_row_0/B:col10/1731278390655/Put/seqid=0 2024-11-10T22:39:51,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742336_1512 (size=12301) 2024-11-10T22:39:51,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-10T22:39:51,143 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T22:39:51,143 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T22:39:51,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] procedure2.ProcedureExecutor(1139): Stored pid=145, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=145, table=TestAcidGuarantees 2024-11-10T22:39:51,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-10T22:39:51,145 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=145, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=145, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T22:39:51,146 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=145, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=145, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T22:39:51,146 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=146, ppid=145, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T22:39:51,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-10T22:39:51,298 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=146 2024-11-10T22:39:51,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f. 2024-11-10T22:39:51,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f. as already flushing 2024-11-10T22:39:51,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f. 2024-11-10T22:39:51,298 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] handler.RSProcedureHandler(58): pid=146 java.io.IOException: Unable to complete flush {ENCODED => ccb0ca518fd65dd7264624eb4a78050f, NAME => 'TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:51,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=146 java.io.IOException: Unable to complete flush {ENCODED => ccb0ca518fd65dd7264624eb4a78050f, NAME => 'TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:51,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=146 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ccb0ca518fd65dd7264624eb4a78050f, NAME => 'TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ccb0ca518fd65dd7264624eb4a78050f, NAME => 'TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:51,394 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:51,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42356 deadline: 1731278451382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:51,394 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:51,394 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:51,394 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:51,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42346 deadline: 1731278451383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:51,394 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:51,395 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:51,395 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:51,395 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:51,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42328 deadline: 1731278451383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:51,395 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:51,395 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:51,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42322 deadline: 1731278451383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:51,395 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:51,395 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:51,395 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:51,395 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:51,395 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:51,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42340 deadline: 1731278451383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:51,395 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:51,395 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:51,396 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:51,396 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:51,396 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:51,451 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=146 2024-11-10T22:39:51,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f. 2024-11-10T22:39:51,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f. as already flushing 2024-11-10T22:39:51,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f. 2024-11-10T22:39:51,452 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] handler.RSProcedureHandler(58): pid=146 java.io.IOException: Unable to complete flush {ENCODED => ccb0ca518fd65dd7264624eb4a78050f, NAME => 'TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:51,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=146 java.io.IOException: Unable to complete flush {ENCODED => ccb0ca518fd65dd7264624eb4a78050f, NAME => 'TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:51,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=146 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ccb0ca518fd65dd7264624eb4a78050f, NAME => 'TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ccb0ca518fd65dd7264624eb4a78050f, NAME => 'TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:51,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-10T22:39:51,489 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=307 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/B/6d60846d3c2c41899e7831d1dc4528c6 2024-11-10T22:39:51,496 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/C/75f3fdf6b01245afa98e3d472b18ed6e is 50, key is test_row_0/C:col10/1731278390655/Put/seqid=0 2024-11-10T22:39:51,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742337_1513 (size=12301) 2024-11-10T22:39:51,604 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=146 2024-11-10T22:39:51,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f. 2024-11-10T22:39:51,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f. as already flushing 2024-11-10T22:39:51,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f. 2024-11-10T22:39:51,605 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] handler.RSProcedureHandler(58): pid=146 java.io.IOException: Unable to complete flush {ENCODED => ccb0ca518fd65dd7264624eb4a78050f, NAME => 'TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:51,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=146 java.io.IOException: Unable to complete flush {ENCODED => ccb0ca518fd65dd7264624eb4a78050f, NAME => 'TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:51,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=146 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ccb0ca518fd65dd7264624eb4a78050f, NAME => 'TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ccb0ca518fd65dd7264624eb4a78050f, NAME => 'TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:51,757 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=146 2024-11-10T22:39:51,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f. 2024-11-10T22:39:51,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f. as already flushing 2024-11-10T22:39:51,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f. 2024-11-10T22:39:51,758 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] handler.RSProcedureHandler(58): pid=146 java.io.IOException: Unable to complete flush {ENCODED => ccb0ca518fd65dd7264624eb4a78050f, NAME => 'TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:51,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=146 java.io.IOException: Unable to complete flush {ENCODED => ccb0ca518fd65dd7264624eb4a78050f, NAME => 'TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:51,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=146 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ccb0ca518fd65dd7264624eb4a78050f, NAME => 'TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ccb0ca518fd65dd7264624eb4a78050f, NAME => 'TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:51,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-10T22:39:51,905 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=307 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/C/75f3fdf6b01245afa98e3d472b18ed6e 2024-11-10T22:39:51,907 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:51,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42356 deadline: 1731278451902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:51,908 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:51,908 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:51,908 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:51,908 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:51,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42346 deadline: 1731278451902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:51,908 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:51,908 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:51,908 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:51,908 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:51,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42340 deadline: 1731278451902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:51,908 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:51,908 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:51,908 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:51,908 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:51,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42328 deadline: 1731278451902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:51,909 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:51,909 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:51,909 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:39:51,909 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:51,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42322 deadline: 1731278451903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 2024-11-10T22:39:51,909 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/A/2784522906f2418c9e206c9218f6afa7 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/A/2784522906f2418c9e206c9218f6afa7 2024-11-10T22:39:51,909 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 , the old value is region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:51,909 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:39:51,909 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., hostname=849d0d71310f,43753,1731278250663, seqNum=2 because the exception is null or not the one we care about 2024-11-10T22:39:51,910 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=146 2024-11-10T22:39:51,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f. 2024-11-10T22:39:51,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f. as already flushing 2024-11-10T22:39:51,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f. 2024-11-10T22:39:51,911 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] handler.RSProcedureHandler(58): pid=146 java.io.IOException: Unable to complete flush {ENCODED => ccb0ca518fd65dd7264624eb4a78050f, NAME => 'TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:51,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=146 java.io.IOException: Unable to complete flush {ENCODED => ccb0ca518fd65dd7264624eb4a78050f, NAME => 'TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:51,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=146 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ccb0ca518fd65dd7264624eb4a78050f, NAME => 'TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ccb0ca518fd65dd7264624eb4a78050f, NAME => 'TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:39:51,913 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/A/2784522906f2418c9e206c9218f6afa7, entries=200, sequenceid=307, filesize=14.4 K 2024-11-10T22:39:51,914 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/B/6d60846d3c2c41899e7831d1dc4528c6 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/B/6d60846d3c2c41899e7831d1dc4528c6 2024-11-10T22:39:51,917 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/B/6d60846d3c2c41899e7831d1dc4528c6, entries=150, sequenceid=307, filesize=12.0 K 2024-11-10T22:39:51,917 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/C/75f3fdf6b01245afa98e3d472b18ed6e as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/C/75f3fdf6b01245afa98e3d472b18ed6e 2024-11-10T22:39:51,920 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/C/75f3fdf6b01245afa98e3d472b18ed6e, entries=150, sequenceid=307, filesize=12.0 K 2024-11-10T22:39:51,921 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for ccb0ca518fd65dd7264624eb4a78050f in 1257ms, sequenceid=307, compaction requested=true 2024-11-10T22:39:51,921 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ccb0ca518fd65dd7264624eb4a78050f: 2024-11-10T22:39:51,921 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ccb0ca518fd65dd7264624eb4a78050f:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T22:39:51,921 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T22:39:51,921 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ccb0ca518fd65dd7264624eb4a78050f:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T22:39:51,921 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T22:39:51,921 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T22:39:51,921 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ccb0ca518fd65dd7264624eb4a78050f:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T22:39:51,921 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T22:39:51,921 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T22:39:51,922 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T22:39:51,922 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39991 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T22:39:51,922 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1541): ccb0ca518fd65dd7264624eb4a78050f/B is initiating minor compaction (all files) 2024-11-10T22:39:51,922 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HStore(1541): ccb0ca518fd65dd7264624eb4a78050f/A is initiating minor compaction (all files) 2024-11-10T22:39:51,922 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ccb0ca518fd65dd7264624eb4a78050f/B in TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f. 2024-11-10T22:39:51,923 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ccb0ca518fd65dd7264624eb4a78050f/A in TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f. 2024-11-10T22:39:51,923 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/B/c2321e6fbcdf40c4851ccfac6223a4ef, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/B/0619c07356fd497d8e316071028ebb56, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/B/6d60846d3c2c41899e7831d1dc4528c6] into tmpdir=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp, totalSize=36.7 K 2024-11-10T22:39:51,923 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/A/7edfff33a077497a83e18e3f9a39d5af, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/A/15d1df3cb18b4b029f7f88f49c3e71d8, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/A/2784522906f2418c9e206c9218f6afa7] into tmpdir=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp, totalSize=39.1 K 2024-11-10T22:39:51,923 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting c2321e6fbcdf40c4851ccfac6223a4ef, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1731278387275 2024-11-10T22:39:51,923 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7edfff33a077497a83e18e3f9a39d5af, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1731278387275 2024-11-10T22:39:51,923 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting 0619c07356fd497d8e316071028ebb56, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1731278388461 2024-11-10T22:39:51,923 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 15d1df3cb18b4b029f7f88f49c3e71d8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1731278388461 2024-11-10T22:39:51,924 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting 6d60846d3c2c41899e7831d1dc4528c6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=307, earliestPutTs=1731278390649 2024-11-10T22:39:51,924 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2784522906f2418c9e206c9218f6afa7, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=307, earliestPutTs=1731278390649 2024-11-10T22:39:51,932 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ccb0ca518fd65dd7264624eb4a78050f#B#compaction#435 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T22:39:51,932 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ccb0ca518fd65dd7264624eb4a78050f#A#compaction#434 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T22:39:51,932 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/B/e5d7683bffa046a1b7c0f4f1711653ce is 50, key is test_row_0/B:col10/1731278390655/Put/seqid=0 2024-11-10T22:39:51,932 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/A/ca50b113475a4477b5a837c4103f7126 is 50, key is test_row_0/A:col10/1731278390655/Put/seqid=0 2024-11-10T22:39:51,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742338_1514 (size=13051) 2024-11-10T22:39:51,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742339_1515 (size=13051) 2024-11-10T22:39:51,953 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/A/ca50b113475a4477b5a837c4103f7126 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/A/ca50b113475a4477b5a837c4103f7126 2024-11-10T22:39:51,956 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/B/e5d7683bffa046a1b7c0f4f1711653ce as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/B/e5d7683bffa046a1b7c0f4f1711653ce 2024-11-10T22:39:51,960 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ccb0ca518fd65dd7264624eb4a78050f/A of ccb0ca518fd65dd7264624eb4a78050f into ca50b113475a4477b5a837c4103f7126(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T22:39:51,960 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ccb0ca518fd65dd7264624eb4a78050f: 2024-11-10T22:39:51,960 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., storeName=ccb0ca518fd65dd7264624eb4a78050f/A, priority=13, startTime=1731278391921; duration=0sec 2024-11-10T22:39:51,960 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T22:39:51,960 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ccb0ca518fd65dd7264624eb4a78050f:A 2024-11-10T22:39:51,960 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ccb0ca518fd65dd7264624eb4a78050f/B of ccb0ca518fd65dd7264624eb4a78050f into e5d7683bffa046a1b7c0f4f1711653ce(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T22:39:51,960 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ccb0ca518fd65dd7264624eb4a78050f: 2024-11-10T22:39:51,960 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., storeName=ccb0ca518fd65dd7264624eb4a78050f/B, priority=13, startTime=1731278391921; duration=0sec 2024-11-10T22:39:51,960 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T22:39:51,960 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T22:39:51,961 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ccb0ca518fd65dd7264624eb4a78050f:B 2024-11-10T22:39:51,961 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T22:39:51,961 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HStore(1541): ccb0ca518fd65dd7264624eb4a78050f/C is initiating minor compaction (all files) 2024-11-10T22:39:51,961 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ccb0ca518fd65dd7264624eb4a78050f/C in TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f. 2024-11-10T22:39:51,961 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/C/d03a6f51ae9d489f887275fdd4ee42df, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/C/3f207e2fe78e4932906fcea2672cdd7c, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/C/75f3fdf6b01245afa98e3d472b18ed6e] into tmpdir=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp, totalSize=36.7 K 2024-11-10T22:39:51,963 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting d03a6f51ae9d489f887275fdd4ee42df, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1731278387275 2024-11-10T22:39:51,964 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3f207e2fe78e4932906fcea2672cdd7c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1731278388461 2024-11-10T22:39:51,964 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 75f3fdf6b01245afa98e3d472b18ed6e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=307, earliestPutTs=1731278390649 2024-11-10T22:39:51,970 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ccb0ca518fd65dd7264624eb4a78050f#C#compaction#436 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T22:39:51,971 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/C/135683ece4574298b51ce7d5db0fc333 is 50, key is test_row_0/C:col10/1731278390655/Put/seqid=0 2024-11-10T22:39:51,976 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742340_1516 (size=13051) 2024-11-10T22:39:52,064 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=146 2024-11-10T22:39:52,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f. 2024-11-10T22:39:52,064 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegion(2902): Flushing ccb0ca518fd65dd7264624eb4a78050f 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-10T22:39:52,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/A/cafcd4727b9e4c6fb5907ab44d63b45c is 50, key is test_row_0/A:col10/1731278390713/Put/seqid=0 2024-11-10T22:39:52,071 INFO [Thread-2017 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2017. 2024-11-10T22:39:52,071 DEBUG [Thread-2017 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T22:39:52,071 DEBUG [Thread-2017 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:39:52,072 DEBUG [Thread-2017 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:39:52,072 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T22:39:52,072 INFO [Thread-2025 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2025. 2024-11-10T22:39:52,072 DEBUG [Thread-2025 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T22:39:52,072 DEBUG [Thread-2025 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:39:52,073 DEBUG [Thread-2025 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:39:52,073 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T22:39:52,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742341_1517 (size=12301) 2024-11-10T22:39:52,075 INFO [Thread-2023 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2023. 2024-11-10T22:39:52,075 DEBUG [Thread-2023 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T22:39:52,075 DEBUG [Thread-2023 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:39:52,075 DEBUG [Thread-2023 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:39:52,075 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T22:39:52,078 INFO [Thread-2021 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2021. 2024-11-10T22:39:52,078 DEBUG [Thread-2021 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T22:39:52,078 DEBUG [Thread-2021 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:39:52,078 DEBUG [Thread-2021 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:39:52,078 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T22:39:52,078 INFO [Thread-2019 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2019. 2024-11-10T22:39:52,078 DEBUG [Thread-2019 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T22:39:52,078 DEBUG [Thread-2019 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:39:52,078 DEBUG [Thread-2019 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:39:52,078 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T22:39:52,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-10T22:39:52,380 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/C/135683ece4574298b51ce7d5db0fc333 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/C/135683ece4574298b51ce7d5db0fc333 2024-11-10T22:39:52,383 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ccb0ca518fd65dd7264624eb4a78050f/C of ccb0ca518fd65dd7264624eb4a78050f into 135683ece4574298b51ce7d5db0fc333(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T22:39:52,383 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ccb0ca518fd65dd7264624eb4a78050f: 2024-11-10T22:39:52,383 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., storeName=ccb0ca518fd65dd7264624eb4a78050f/C, priority=13, startTime=1731278391921; duration=0sec 2024-11-10T22:39:52,383 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T22:39:52,383 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ccb0ca518fd65dd7264624eb4a78050f:C 2024-11-10T22:39:52,474 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=334 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/A/cafcd4727b9e4c6fb5907ab44d63b45c 2024-11-10T22:39:52,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/B/bf237e8cc0474e1581381826edf9da8a is 50, key is test_row_0/B:col10/1731278390713/Put/seqid=0 2024-11-10T22:39:52,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742342_1518 (size=12301) 2024-11-10T22:39:52,883 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=334 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/B/bf237e8cc0474e1581381826edf9da8a 2024-11-10T22:39:52,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/C/13fdb09da46f4eb5a947435f3f469775 is 50, key is test_row_0/C:col10/1731278390713/Put/seqid=0 2024-11-10T22:39:52,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742343_1519 (size=12301) 2024-11-10T22:39:52,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(8855): Flush requested on ccb0ca518fd65dd7264624eb4a78050f 2024-11-10T22:39:52,914 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f. as already flushing 2024-11-10T22:39:52,914 INFO [Thread-2012 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2012. 2024-11-10T22:39:52,914 DEBUG [Thread-2012 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T22:39:52,914 DEBUG [Thread-2012 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:39:52,914 INFO [Thread-2014 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2014. 2024-11-10T22:39:52,914 DEBUG [Thread-2014 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T22:39:52,914 DEBUG [Thread-2014 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:39:52,914 DEBUG [Thread-2012 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:39:52,914 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T22:39:52,914 DEBUG [Thread-2014 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:39:52,914 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T22:39:52,924 INFO [Thread-2008 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2008. 2024-11-10T22:39:52,924 INFO [Thread-2006 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2006. 2024-11-10T22:39:52,924 DEBUG [Thread-2008 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T22:39:52,924 INFO [Thread-2010 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2010. 2024-11-10T22:39:52,924 DEBUG [Thread-2008 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:39:52,924 DEBUG [Thread-2006 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T22:39:52,924 DEBUG [Thread-2010 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T22:39:52,925 DEBUG [Thread-2010 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:39:52,925 DEBUG [Thread-2006 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:39:52,925 DEBUG [Thread-2008 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:39:52,925 DEBUG [Thread-2010 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:39:52,925 DEBUG [Thread-2006 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:39:52,925 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T22:39:52,925 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T22:39:52,925 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T22:39:53,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-10T22:39:53,293 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=334 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/C/13fdb09da46f4eb5a947435f3f469775 2024-11-10T22:39:53,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/A/cafcd4727b9e4c6fb5907ab44d63b45c as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/A/cafcd4727b9e4c6fb5907ab44d63b45c 2024-11-10T22:39:53,299 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/A/cafcd4727b9e4c6fb5907ab44d63b45c, entries=150, sequenceid=334, filesize=12.0 K 2024-11-10T22:39:53,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/B/bf237e8cc0474e1581381826edf9da8a as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/B/bf237e8cc0474e1581381826edf9da8a 2024-11-10T22:39:53,303 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/B/bf237e8cc0474e1581381826edf9da8a, entries=150, sequenceid=334, filesize=12.0 K 2024-11-10T22:39:53,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/C/13fdb09da46f4eb5a947435f3f469775 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/C/13fdb09da46f4eb5a947435f3f469775 2024-11-10T22:39:53,306 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/C/13fdb09da46f4eb5a947435f3f469775, entries=150, sequenceid=334, filesize=12.0 K 2024-11-10T22:39:53,307 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=33.54 KB/34350 for ccb0ca518fd65dd7264624eb4a78050f in 1243ms, sequenceid=334, compaction requested=false 2024-11-10T22:39:53,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegion(2603): Flush status journal for ccb0ca518fd65dd7264624eb4a78050f: 2024-11-10T22:39:53,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f. 2024-11-10T22:39:53,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=146 2024-11-10T22:39:53,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.HMaster(4169): Remote procedure done, pid=146 2024-11-10T22:39:53,311 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=146, resume processing ppid=145 2024-11-10T22:39:53,311 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, ppid=145, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1630 sec 2024-11-10T22:39:53,312 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=145, table=TestAcidGuarantees in 2.1680 sec 2024-11-10T22:39:55,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-10T22:39:55,302 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T22:39:55,303 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-10T22:39:55,303 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 57 2024-11-10T22:39:55,303 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 56 2024-11-10T22:39:55,303 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 51 2024-11-10T22:39:55,303 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 42 2024-11-10T22:39:55,303 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 61 2024-11-10T22:39:55,303 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-10T22:39:55,303 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-10T22:39:55,303 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1212 2024-11-10T22:39:55,303 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3630 rows 2024-11-10T22:39:55,303 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1213 2024-11-10T22:39:55,303 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3639 rows 2024-11-10T22:39:55,303 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1214 2024-11-10T22:39:55,303 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3641 rows 2024-11-10T22:39:55,303 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1208 2024-11-10T22:39:55,303 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3624 rows 2024-11-10T22:39:55,303 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1218 2024-11-10T22:39:55,303 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3653 rows 2024-11-10T22:39:55,303 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-10T22:39:55,303 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T22:39:55,303 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:39:55,303 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:39:55,303 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T22:39:55,304 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-10T22:39:55,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] procedure2.ProcedureExecutor(1139): Stored pid=147, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-10T22:39:55,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=147 2024-11-10T22:39:55,307 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731278395307"}]},"ts":"1731278395307"} 2024-11-10T22:39:55,308 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-10T22:39:55,308 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-10T22:39:55,309 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=148, ppid=147, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-10T22:39:55,310 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ccb0ca518fd65dd7264624eb4a78050f, UNASSIGN}] 2024-11-10T22:39:55,310 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ccb0ca518fd65dd7264624eb4a78050f, UNASSIGN 2024-11-10T22:39:55,311 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=149 updating hbase:meta row=ccb0ca518fd65dd7264624eb4a78050f, regionState=CLOSING, regionLocation=849d0d71310f,43753,1731278250663 2024-11-10T22:39:55,312 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ccb0ca518fd65dd7264624eb4a78050f, UNASSIGN because future has completed 2024-11-10T22:39:55,312 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-10T22:39:55,312 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=150, ppid=149, state=RUNNABLE, hasLock=false; CloseRegionProcedure ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663}] 2024-11-10T22:39:55,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=147 2024-11-10T22:39:55,465 INFO [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] handler.UnassignRegionHandler(122): Close ccb0ca518fd65dd7264624eb4a78050f 2024-11-10T22:39:55,465 DEBUG [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-10T22:39:55,465 DEBUG [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1722): Closing ccb0ca518fd65dd7264624eb4a78050f, disabling compactions & flushes 2024-11-10T22:39:55,465 INFO [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f. 2024-11-10T22:39:55,465 DEBUG [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f. 2024-11-10T22:39:55,465 DEBUG [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f. after waiting 0 ms 2024-11-10T22:39:55,465 DEBUG [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f. 2024-11-10T22:39:55,465 INFO [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(2902): Flushing ccb0ca518fd65dd7264624eb4a78050f 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-10T22:39:55,468 DEBUG [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/A/511d38d4d64a426eb2985d3939b42c98 is 50, key is test_row_0/A:col10/1731278392923/Put/seqid=0 2024-11-10T22:39:55,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742344_1520 (size=12301) 2024-11-10T22:39:55,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=147 2024-11-10T22:39:55,872 INFO [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=343 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/A/511d38d4d64a426eb2985d3939b42c98 2024-11-10T22:39:55,878 DEBUG [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/B/149ad42e8932450d891a734fd4448662 is 50, key is test_row_0/B:col10/1731278392923/Put/seqid=0 2024-11-10T22:39:55,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742345_1521 (size=12301) 2024-11-10T22:39:55,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=147 2024-11-10T22:39:56,281 INFO [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=343 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/B/149ad42e8932450d891a734fd4448662 2024-11-10T22:39:56,286 DEBUG [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/C/afa4dd127f284f91ba8a3030f29b4517 is 50, key is test_row_0/C:col10/1731278392923/Put/seqid=0 2024-11-10T22:39:56,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742346_1522 (size=12301) 2024-11-10T22:39:56,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=147 2024-11-10T22:39:56,690 INFO [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=343 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/C/afa4dd127f284f91ba8a3030f29b4517 2024-11-10T22:39:56,693 DEBUG [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/A/511d38d4d64a426eb2985d3939b42c98 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/A/511d38d4d64a426eb2985d3939b42c98 2024-11-10T22:39:56,696 INFO [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/A/511d38d4d64a426eb2985d3939b42c98, entries=150, sequenceid=343, filesize=12.0 K 2024-11-10T22:39:56,697 DEBUG [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/B/149ad42e8932450d891a734fd4448662 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/B/149ad42e8932450d891a734fd4448662 2024-11-10T22:39:56,699 INFO [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/B/149ad42e8932450d891a734fd4448662, entries=150, sequenceid=343, filesize=12.0 K 2024-11-10T22:39:56,700 DEBUG [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/.tmp/C/afa4dd127f284f91ba8a3030f29b4517 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/C/afa4dd127f284f91ba8a3030f29b4517 2024-11-10T22:39:56,702 INFO [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/C/afa4dd127f284f91ba8a3030f29b4517, entries=150, sequenceid=343, filesize=12.0 K 2024-11-10T22:39:56,703 INFO [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for ccb0ca518fd65dd7264624eb4a78050f in 1238ms, sequenceid=343, compaction requested=true 2024-11-10T22:39:56,703 DEBUG [StoreCloser-TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/A/279bda6940fd43d68f05566362ff9f78, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/A/2b54f76a241245999048e3f8c0df72ad, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/A/47627124fced434189b9fb0d529b4b50, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/A/b994f1f9ef4044b58412dabe05596407, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/A/6acb83a744eb48678bb74cb2de683dec, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/A/e7914d9aa25447c9a1e3cd08a5a1cad0, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/A/0bacdee9eb9b4185bd5175357adc09e6, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/A/c3a460221f4648f686db1bcb736c1cf9, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/A/c520cf2332b84164a1760bd4108814fc, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/A/9bc522c3d9b447399cce6c192ac3d293, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/A/6e237ee03183468cb7cf9f8d95f49742, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/A/4f39b2de2ad24eec956bce0b5442f63c, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/A/341c7132a0954a5294c7009c62c4b5f6, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/A/139d13daf2d54f1f8551a37b92c363c4, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/A/da3cd2f914254827a25cad0585196af7, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/A/3677e016e8864fa2ad2b7054e695491a, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/A/91ebb218427b4926a65143dad7f2285c, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/A/2fd383d277644494ba599019cf02b141, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/A/aef42e86ee384591a0a6939fe3226347, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/A/7edfff33a077497a83e18e3f9a39d5af, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/A/15d1df3cb18b4b029f7f88f49c3e71d8, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/A/2784522906f2418c9e206c9218f6afa7] to archive 2024-11-10T22:39:56,704 DEBUG [StoreCloser-TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T22:39:56,705 DEBUG [StoreCloser-TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/A/279bda6940fd43d68f05566362ff9f78 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/A/279bda6940fd43d68f05566362ff9f78 2024-11-10T22:39:56,706 DEBUG [StoreCloser-TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/A/2b54f76a241245999048e3f8c0df72ad to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/A/2b54f76a241245999048e3f8c0df72ad 2024-11-10T22:39:56,707 DEBUG [StoreCloser-TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/A/47627124fced434189b9fb0d529b4b50 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/A/47627124fced434189b9fb0d529b4b50 2024-11-10T22:39:56,708 DEBUG [StoreCloser-TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/A/b994f1f9ef4044b58412dabe05596407 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/A/b994f1f9ef4044b58412dabe05596407 2024-11-10T22:39:56,708 DEBUG [StoreCloser-TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/A/6acb83a744eb48678bb74cb2de683dec to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/A/6acb83a744eb48678bb74cb2de683dec 2024-11-10T22:39:56,709 DEBUG [StoreCloser-TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/A/e7914d9aa25447c9a1e3cd08a5a1cad0 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/A/e7914d9aa25447c9a1e3cd08a5a1cad0 2024-11-10T22:39:56,710 DEBUG [StoreCloser-TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/A/0bacdee9eb9b4185bd5175357adc09e6 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/A/0bacdee9eb9b4185bd5175357adc09e6 2024-11-10T22:39:56,711 DEBUG [StoreCloser-TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/A/c3a460221f4648f686db1bcb736c1cf9 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/A/c3a460221f4648f686db1bcb736c1cf9 2024-11-10T22:39:56,711 DEBUG [StoreCloser-TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/A/c520cf2332b84164a1760bd4108814fc to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/A/c520cf2332b84164a1760bd4108814fc 2024-11-10T22:39:56,712 DEBUG [StoreCloser-TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/A/9bc522c3d9b447399cce6c192ac3d293 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/A/9bc522c3d9b447399cce6c192ac3d293 2024-11-10T22:39:56,713 DEBUG [StoreCloser-TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/A/6e237ee03183468cb7cf9f8d95f49742 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/A/6e237ee03183468cb7cf9f8d95f49742 2024-11-10T22:39:56,714 DEBUG [StoreCloser-TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/A/4f39b2de2ad24eec956bce0b5442f63c to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/A/4f39b2de2ad24eec956bce0b5442f63c 2024-11-10T22:39:56,714 DEBUG [StoreCloser-TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/A/341c7132a0954a5294c7009c62c4b5f6 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/A/341c7132a0954a5294c7009c62c4b5f6 2024-11-10T22:39:56,715 DEBUG [StoreCloser-TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/A/139d13daf2d54f1f8551a37b92c363c4 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/A/139d13daf2d54f1f8551a37b92c363c4 2024-11-10T22:39:56,716 DEBUG [StoreCloser-TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/A/da3cd2f914254827a25cad0585196af7 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/A/da3cd2f914254827a25cad0585196af7 2024-11-10T22:39:56,717 DEBUG [StoreCloser-TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/A/3677e016e8864fa2ad2b7054e695491a to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/A/3677e016e8864fa2ad2b7054e695491a 2024-11-10T22:39:56,717 DEBUG [StoreCloser-TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/A/91ebb218427b4926a65143dad7f2285c to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/A/91ebb218427b4926a65143dad7f2285c 2024-11-10T22:39:56,718 DEBUG [StoreCloser-TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/A/2fd383d277644494ba599019cf02b141 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/A/2fd383d277644494ba599019cf02b141 2024-11-10T22:39:56,719 DEBUG [StoreCloser-TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/A/aef42e86ee384591a0a6939fe3226347 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/A/aef42e86ee384591a0a6939fe3226347 2024-11-10T22:39:56,720 DEBUG [StoreCloser-TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/A/7edfff33a077497a83e18e3f9a39d5af to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/A/7edfff33a077497a83e18e3f9a39d5af 2024-11-10T22:39:56,720 DEBUG [StoreCloser-TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/A/15d1df3cb18b4b029f7f88f49c3e71d8 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/A/15d1df3cb18b4b029f7f88f49c3e71d8 2024-11-10T22:39:56,721 DEBUG [StoreCloser-TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/A/2784522906f2418c9e206c9218f6afa7 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/A/2784522906f2418c9e206c9218f6afa7 2024-11-10T22:39:56,722 DEBUG [StoreCloser-TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/B/aa0700e4a9234c1b8380465d65e362c7, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/B/c54800d2f8e44045b2c9356b70df9e50, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/B/bbcc8c5520574952a4cffb32a0653ba4, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/B/276a98a5b3b2416aa59dbc2fa649b324, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/B/729255f0edd94ba59be3212284a86704, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/B/0639e5ec3f0f4efc91e7367515c86882, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/B/e8655bdb107c4710b0999f7d4e50bea5, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/B/f81cc73ebbea452eb141b8e8f15599bc, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/B/034ec481bc0c46d4bb8a1c26b4c7e01d, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/B/234c945f6b004debad9f25609573ee8a, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/B/3cc1419f79fa4f74b537a8897d30b4fe, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/B/ed048410aa224771829b127492153484, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/B/2e149c2a7a5c4d4ebd647a115161e201, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/B/f728c467bea34a3090715001e84b183b, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/B/c0c6e5a65e304bc497a536ee5a1a5cc1, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/B/e07eaacdac0b43309977ecb4d30ae2cb, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/B/84e06cbedab5468db22af82a2e00c4ff, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/B/e1a484f4837c446cbc1e49a012727be8, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/B/c2321e6fbcdf40c4851ccfac6223a4ef, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/B/4455656a67eb430e898492f51897ec9b, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/B/0619c07356fd497d8e316071028ebb56, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/B/6d60846d3c2c41899e7831d1dc4528c6] to archive 2024-11-10T22:39:56,723 DEBUG [StoreCloser-TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T22:39:56,724 DEBUG [StoreCloser-TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/B/aa0700e4a9234c1b8380465d65e362c7 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/B/aa0700e4a9234c1b8380465d65e362c7 2024-11-10T22:39:56,725 DEBUG [StoreCloser-TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/B/c54800d2f8e44045b2c9356b70df9e50 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/B/c54800d2f8e44045b2c9356b70df9e50 2024-11-10T22:39:56,726 DEBUG [StoreCloser-TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/B/bbcc8c5520574952a4cffb32a0653ba4 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/B/bbcc8c5520574952a4cffb32a0653ba4 2024-11-10T22:39:56,727 DEBUG [StoreCloser-TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/B/276a98a5b3b2416aa59dbc2fa649b324 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/B/276a98a5b3b2416aa59dbc2fa649b324 2024-11-10T22:39:56,727 DEBUG [StoreCloser-TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/B/729255f0edd94ba59be3212284a86704 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/B/729255f0edd94ba59be3212284a86704 2024-11-10T22:39:56,728 DEBUG [StoreCloser-TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/B/0639e5ec3f0f4efc91e7367515c86882 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/B/0639e5ec3f0f4efc91e7367515c86882 2024-11-10T22:39:56,729 DEBUG [StoreCloser-TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/B/e8655bdb107c4710b0999f7d4e50bea5 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/B/e8655bdb107c4710b0999f7d4e50bea5 2024-11-10T22:39:56,730 DEBUG [StoreCloser-TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/B/f81cc73ebbea452eb141b8e8f15599bc to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/B/f81cc73ebbea452eb141b8e8f15599bc 2024-11-10T22:39:56,731 DEBUG [StoreCloser-TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/B/034ec481bc0c46d4bb8a1c26b4c7e01d to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/B/034ec481bc0c46d4bb8a1c26b4c7e01d 2024-11-10T22:39:56,732 DEBUG [StoreCloser-TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/B/234c945f6b004debad9f25609573ee8a to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/B/234c945f6b004debad9f25609573ee8a 2024-11-10T22:39:56,732 DEBUG [StoreCloser-TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/B/3cc1419f79fa4f74b537a8897d30b4fe to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/B/3cc1419f79fa4f74b537a8897d30b4fe 2024-11-10T22:39:56,733 DEBUG [StoreCloser-TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/B/ed048410aa224771829b127492153484 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/B/ed048410aa224771829b127492153484 2024-11-10T22:39:56,734 DEBUG [StoreCloser-TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/B/2e149c2a7a5c4d4ebd647a115161e201 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/B/2e149c2a7a5c4d4ebd647a115161e201 2024-11-10T22:39:56,735 DEBUG [StoreCloser-TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/B/f728c467bea34a3090715001e84b183b to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/B/f728c467bea34a3090715001e84b183b 2024-11-10T22:39:56,736 DEBUG [StoreCloser-TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/B/c0c6e5a65e304bc497a536ee5a1a5cc1 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/B/c0c6e5a65e304bc497a536ee5a1a5cc1 2024-11-10T22:39:56,737 DEBUG [StoreCloser-TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/B/e07eaacdac0b43309977ecb4d30ae2cb to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/B/e07eaacdac0b43309977ecb4d30ae2cb 2024-11-10T22:39:56,738 DEBUG [StoreCloser-TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/B/84e06cbedab5468db22af82a2e00c4ff to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/B/84e06cbedab5468db22af82a2e00c4ff 2024-11-10T22:39:56,738 DEBUG [StoreCloser-TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/B/e1a484f4837c446cbc1e49a012727be8 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/B/e1a484f4837c446cbc1e49a012727be8 2024-11-10T22:39:56,739 DEBUG [StoreCloser-TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/B/c2321e6fbcdf40c4851ccfac6223a4ef to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/B/c2321e6fbcdf40c4851ccfac6223a4ef 2024-11-10T22:39:56,740 DEBUG [StoreCloser-TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/B/4455656a67eb430e898492f51897ec9b to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/B/4455656a67eb430e898492f51897ec9b 2024-11-10T22:39:56,741 DEBUG [StoreCloser-TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/B/0619c07356fd497d8e316071028ebb56 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/B/0619c07356fd497d8e316071028ebb56 2024-11-10T22:39:56,742 DEBUG [StoreCloser-TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/B/6d60846d3c2c41899e7831d1dc4528c6 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/B/6d60846d3c2c41899e7831d1dc4528c6 2024-11-10T22:39:56,743 DEBUG [StoreCloser-TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/C/87d429300f2d47e59f9ffcf4de4d6a61, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/C/5fbd6182c0ee4b45b8388005d0d3de5a, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/C/82006ec117f64b21b03a7c6d03992925, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/C/a75f3b7fbeb14c5f80574cad7f8d9c1f, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/C/c7af908f2d49400faef2fbde9d3037d0, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/C/e237887b7a394e15b0e2418f929f60a3, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/C/63a7980c1c7344d1be2f91cdf247ccde, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/C/f24fc557f2cb4b63aef92b6f3a678c8c, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/C/53267623b9c44abfbae788aaf65f6df2, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/C/249507aa08374e0d87ad7b5e55ccaf46, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/C/d5759639fde545c0a08ad1f2d005f6ef, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/C/d02b83d77ad441d9b838760590ef8caa, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/C/4e9b2e5fa195429e9549c94617321513, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/C/3a7e8c5f028a401ea14a44aae8911df9, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/C/c6c25768bb104ff58c6c08c097d4cfa2, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/C/0dc7102c2e914874808b456945580fdb, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/C/b8ee7fc7e2f94ae2bc4dbad6a13774be, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/C/62d497f73010486893406b9d9d94b349, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/C/d03a6f51ae9d489f887275fdd4ee42df, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/C/fbe195f9ef534ffa9e1526e73df2edd4, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/C/3f207e2fe78e4932906fcea2672cdd7c, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/C/75f3fdf6b01245afa98e3d472b18ed6e] to archive 2024-11-10T22:39:56,744 DEBUG [StoreCloser-TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T22:39:56,745 DEBUG [StoreCloser-TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/C/87d429300f2d47e59f9ffcf4de4d6a61 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/C/87d429300f2d47e59f9ffcf4de4d6a61 2024-11-10T22:39:56,746 DEBUG [StoreCloser-TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/C/5fbd6182c0ee4b45b8388005d0d3de5a to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/C/5fbd6182c0ee4b45b8388005d0d3de5a 2024-11-10T22:39:56,747 DEBUG [StoreCloser-TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/C/82006ec117f64b21b03a7c6d03992925 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/C/82006ec117f64b21b03a7c6d03992925 2024-11-10T22:39:56,748 DEBUG [StoreCloser-TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/C/a75f3b7fbeb14c5f80574cad7f8d9c1f to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/C/a75f3b7fbeb14c5f80574cad7f8d9c1f 2024-11-10T22:39:56,748 DEBUG [StoreCloser-TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/C/c7af908f2d49400faef2fbde9d3037d0 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/C/c7af908f2d49400faef2fbde9d3037d0 2024-11-10T22:39:56,749 DEBUG [StoreCloser-TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/C/e237887b7a394e15b0e2418f929f60a3 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/C/e237887b7a394e15b0e2418f929f60a3 2024-11-10T22:39:56,750 DEBUG [StoreCloser-TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/C/63a7980c1c7344d1be2f91cdf247ccde to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/C/63a7980c1c7344d1be2f91cdf247ccde 2024-11-10T22:39:56,751 DEBUG [StoreCloser-TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/C/f24fc557f2cb4b63aef92b6f3a678c8c to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/C/f24fc557f2cb4b63aef92b6f3a678c8c 2024-11-10T22:39:56,752 DEBUG [StoreCloser-TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/C/53267623b9c44abfbae788aaf65f6df2 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/C/53267623b9c44abfbae788aaf65f6df2 2024-11-10T22:39:56,752 DEBUG [StoreCloser-TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/C/249507aa08374e0d87ad7b5e55ccaf46 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/C/249507aa08374e0d87ad7b5e55ccaf46 2024-11-10T22:39:56,753 DEBUG [StoreCloser-TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/C/d5759639fde545c0a08ad1f2d005f6ef to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/C/d5759639fde545c0a08ad1f2d005f6ef 2024-11-10T22:39:56,754 DEBUG [StoreCloser-TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/C/d02b83d77ad441d9b838760590ef8caa to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/C/d02b83d77ad441d9b838760590ef8caa 2024-11-10T22:39:56,755 DEBUG [StoreCloser-TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/C/4e9b2e5fa195429e9549c94617321513 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/C/4e9b2e5fa195429e9549c94617321513 2024-11-10T22:39:56,755 DEBUG [StoreCloser-TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/C/3a7e8c5f028a401ea14a44aae8911df9 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/C/3a7e8c5f028a401ea14a44aae8911df9 2024-11-10T22:39:56,756 DEBUG [StoreCloser-TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/C/c6c25768bb104ff58c6c08c097d4cfa2 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/C/c6c25768bb104ff58c6c08c097d4cfa2 2024-11-10T22:39:56,757 DEBUG [StoreCloser-TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/C/0dc7102c2e914874808b456945580fdb to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/C/0dc7102c2e914874808b456945580fdb 2024-11-10T22:39:56,757 DEBUG [StoreCloser-TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/C/b8ee7fc7e2f94ae2bc4dbad6a13774be to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/C/b8ee7fc7e2f94ae2bc4dbad6a13774be 2024-11-10T22:39:56,758 DEBUG [StoreCloser-TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/C/62d497f73010486893406b9d9d94b349 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/C/62d497f73010486893406b9d9d94b349 2024-11-10T22:39:56,759 DEBUG [StoreCloser-TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/C/d03a6f51ae9d489f887275fdd4ee42df to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/C/d03a6f51ae9d489f887275fdd4ee42df 2024-11-10T22:39:56,760 DEBUG [StoreCloser-TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/C/fbe195f9ef534ffa9e1526e73df2edd4 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/C/fbe195f9ef534ffa9e1526e73df2edd4 2024-11-10T22:39:56,761 DEBUG [StoreCloser-TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/C/3f207e2fe78e4932906fcea2672cdd7c to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/C/3f207e2fe78e4932906fcea2672cdd7c 2024-11-10T22:39:56,761 DEBUG [StoreCloser-TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/C/75f3fdf6b01245afa98e3d472b18ed6e to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/C/75f3fdf6b01245afa98e3d472b18ed6e 2024-11-10T22:39:56,765 DEBUG [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/recovered.edits/346.seqid, newMaxSeqId=346, maxSeqId=1 2024-11-10T22:39:56,766 INFO [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f. 2024-11-10T22:39:56,766 DEBUG [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1676): Region close journal for ccb0ca518fd65dd7264624eb4a78050f: Waiting for close lock at 1731278395465Running coprocessor pre-close hooks at 1731278395465Disabling compacts and flushes for region at 1731278395465Disabling writes for close at 1731278395465Obtaining lock to block concurrent updates at 1731278395465Preparing flush snapshotting stores in ccb0ca518fd65dd7264624eb4a78050f at 1731278395465Finished memstore snapshotting TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1731278395465Flushing stores of TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f. at 1731278395466 (+1 ms)Flushing ccb0ca518fd65dd7264624eb4a78050f/A: creating writer at 1731278395466Flushing ccb0ca518fd65dd7264624eb4a78050f/A: appending metadata at 1731278395468 (+2 ms)Flushing ccb0ca518fd65dd7264624eb4a78050f/A: closing flushed file at 1731278395468Flushing ccb0ca518fd65dd7264624eb4a78050f/B: creating writer at 1731278395875 (+407 ms)Flushing ccb0ca518fd65dd7264624eb4a78050f/B: appending metadata at 1731278395877 (+2 ms)Flushing ccb0ca518fd65dd7264624eb4a78050f/B: closing flushed file at 1731278395877Flushing ccb0ca518fd65dd7264624eb4a78050f/C: creating writer at 1731278396284 (+407 ms)Flushing ccb0ca518fd65dd7264624eb4a78050f/C: appending metadata at 1731278396286 (+2 ms)Flushing ccb0ca518fd65dd7264624eb4a78050f/C: closing flushed file at 1731278396286Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1f017ea: reopening flushed file at 1731278396693 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@e6e7238: reopening flushed file at 1731278396696 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2c21d8dd: reopening flushed file at 1731278396699 (+3 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for ccb0ca518fd65dd7264624eb4a78050f in 1238ms, sequenceid=343, compaction requested=true at 1731278396703 (+4 ms)Writing region close event to WAL at 1731278396763 (+60 ms)Running coprocessor post-close hooks at 1731278396766 (+3 ms)Closed at 1731278396766 2024-11-10T22:39:56,768 INFO [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] handler.UnassignRegionHandler(157): Closed ccb0ca518fd65dd7264624eb4a78050f 2024-11-10T22:39:56,768 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=149 updating hbase:meta row=ccb0ca518fd65dd7264624eb4a78050f, regionState=CLOSED 2024-11-10T22:39:56,769 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=150, ppid=149, state=RUNNABLE, hasLock=false; CloseRegionProcedure ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 because future has completed 2024-11-10T22:39:56,771 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=150, resume processing ppid=149 2024-11-10T22:39:56,771 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, ppid=149, state=SUCCESS, hasLock=false; CloseRegionProcedure ccb0ca518fd65dd7264624eb4a78050f, server=849d0d71310f,43753,1731278250663 in 1.4580 sec 2024-11-10T22:39:56,773 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=149, resume processing ppid=148 2024-11-10T22:39:56,773 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, ppid=148, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ccb0ca518fd65dd7264624eb4a78050f, UNASSIGN in 1.4610 sec 2024-11-10T22:39:56,774 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=148, resume processing ppid=147 2024-11-10T22:39:56,774 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, ppid=147, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.4640 sec 2024-11-10T22:39:56,775 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731278396775"}]},"ts":"1731278396775"} 2024-11-10T22:39:56,777 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-10T22:39:56,777 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-10T22:39:56,778 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.4730 sec 2024-11-10T22:39:57,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=147 2024-11-10T22:39:57,452 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-10T22:39:57,453 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-10T22:39:57,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] procedure2.ProcedureExecutor(1139): Stored pid=151, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T22:39:57,454 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=151, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T22:39:57,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=151 2024-11-10T22:39:57,455 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=151, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T22:39:57,457 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f 2024-11-10T22:39:57,459 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/A, FileablePath, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/B, FileablePath, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/C, FileablePath, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/recovered.edits] 2024-11-10T22:39:57,461 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/A/511d38d4d64a426eb2985d3939b42c98 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/A/511d38d4d64a426eb2985d3939b42c98 2024-11-10T22:39:57,462 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/A/ca50b113475a4477b5a837c4103f7126 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/A/ca50b113475a4477b5a837c4103f7126 2024-11-10T22:39:57,463 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/A/cafcd4727b9e4c6fb5907ab44d63b45c to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/A/cafcd4727b9e4c6fb5907ab44d63b45c 2024-11-10T22:39:57,465 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/B/149ad42e8932450d891a734fd4448662 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/B/149ad42e8932450d891a734fd4448662 2024-11-10T22:39:57,466 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/B/bf237e8cc0474e1581381826edf9da8a to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/B/bf237e8cc0474e1581381826edf9da8a 2024-11-10T22:39:57,467 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/B/e5d7683bffa046a1b7c0f4f1711653ce to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/B/e5d7683bffa046a1b7c0f4f1711653ce 2024-11-10T22:39:57,468 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/C/135683ece4574298b51ce7d5db0fc333 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/C/135683ece4574298b51ce7d5db0fc333 2024-11-10T22:39:57,469 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/C/13fdb09da46f4eb5a947435f3f469775 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/C/13fdb09da46f4eb5a947435f3f469775 2024-11-10T22:39:57,470 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/C/afa4dd127f284f91ba8a3030f29b4517 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/C/afa4dd127f284f91ba8a3030f29b4517 2024-11-10T22:39:57,472 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/recovered.edits/346.seqid to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f/recovered.edits/346.seqid 2024-11-10T22:39:57,473 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/ccb0ca518fd65dd7264624eb4a78050f 2024-11-10T22:39:57,473 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-10T22:39:57,474 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=151, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T22:39:57,476 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-10T22:39:57,478 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-10T22:39:57,479 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=151, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T22:39:57,479 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-10T22:39:57,479 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731278397479"}]},"ts":"9223372036854775807"} 2024-11-10T22:39:57,481 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-10T22:39:57,481 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => ccb0ca518fd65dd7264624eb4a78050f, NAME => 'TestAcidGuarantees,,1731278369880.ccb0ca518fd65dd7264624eb4a78050f.', STARTKEY => '', ENDKEY => ''}] 2024-11-10T22:39:57,481 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-10T22:39:57,481 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731278397481"}]},"ts":"9223372036854775807"} 2024-11-10T22:39:57,482 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-10T22:39:57,483 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=151, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T22:39:57,483 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 30 msec 2024-11-10T22:39:57,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=151 2024-11-10T22:39:57,563 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-10T22:39:57,563 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-10T22:39:57,573 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testScanAtomicity Thread=240 (was 238) - Thread LEAK? -, OpenFileDescriptor=449 (was 453), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=275 (was 284), ProcessCount=11 (was 11), AvailableMemoryMB=6927 (was 6951) 2024-11-10T22:39:57,581 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobGetAtomicity Thread=240, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=275, ProcessCount=11, AvailableMemoryMB=6927 2024-11-10T22:39:57,582 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-10T22:39:57,583 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-10T22:39:57,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] procedure2.ProcedureExecutor(1139): Stored pid=152, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-10T22:39:57,584 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=152, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-10T22:39:57,584 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:57,584 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 152 2024-11-10T22:39:57,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-10T22:39:57,585 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=152, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-10T22:39:57,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742347_1523 (size=959) 2024-11-10T22:39:57,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-10T22:39:57,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-10T22:39:57,991 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 74244978b603a65b5f2074c624cb7957, NAME => 'TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39 2024-11-10T22:39:57,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742348_1524 (size=53) 2024-11-10T22:39:58,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-10T22:39:58,396 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T22:39:58,396 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 74244978b603a65b5f2074c624cb7957, disabling compactions & flushes 2024-11-10T22:39:58,396 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. 2024-11-10T22:39:58,396 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. 2024-11-10T22:39:58,396 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. after waiting 0 ms 2024-11-10T22:39:58,396 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. 2024-11-10T22:39:58,396 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. 2024-11-10T22:39:58,396 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 74244978b603a65b5f2074c624cb7957: Waiting for close lock at 1731278398396Disabling compacts and flushes for region at 1731278398396Disabling writes for close at 1731278398396Writing region close event to WAL at 1731278398396Closed at 1731278398396 2024-11-10T22:39:58,397 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=152, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-10T22:39:58,398 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731278398397"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731278398397"}]},"ts":"1731278398397"} 2024-11-10T22:39:58,399 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-10T22:39:58,400 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=152, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-10T22:39:58,400 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731278398400"}]},"ts":"1731278398400"} 2024-11-10T22:39:58,401 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-10T22:39:58,401 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=152, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=74244978b603a65b5f2074c624cb7957, ASSIGN}] 2024-11-10T22:39:58,402 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=153, ppid=152, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=74244978b603a65b5f2074c624cb7957, ASSIGN 2024-11-10T22:39:58,402 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=153, ppid=152, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=74244978b603a65b5f2074c624cb7957, ASSIGN; state=OFFLINE, location=849d0d71310f,43753,1731278250663; forceNewPlan=false, retain=false 2024-11-10T22:39:58,553 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=153 updating hbase:meta row=74244978b603a65b5f2074c624cb7957, regionState=OPENING, regionLocation=849d0d71310f,43753,1731278250663 2024-11-10T22:39:58,555 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=153, ppid=152, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=74244978b603a65b5f2074c624cb7957, ASSIGN because future has completed 2024-11-10T22:39:58,555 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=154, ppid=153, state=RUNNABLE, hasLock=false; OpenRegionProcedure 74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663}] 2024-11-10T22:39:58,709 INFO [RS_OPEN_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. 2024-11-10T22:39:58,709 DEBUG [RS_OPEN_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.HRegion(7752): Opening region: {ENCODED => 74244978b603a65b5f2074c624cb7957, NAME => 'TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.', STARTKEY => '', ENDKEY => ''} 2024-11-10T22:39:58,710 DEBUG [RS_OPEN_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 74244978b603a65b5f2074c624cb7957 2024-11-10T22:39:58,710 DEBUG [RS_OPEN_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T22:39:58,710 DEBUG [RS_OPEN_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.HRegion(7794): checking encryption for 74244978b603a65b5f2074c624cb7957 2024-11-10T22:39:58,710 DEBUG [RS_OPEN_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.HRegion(7797): checking classloading for 74244978b603a65b5f2074c624cb7957 2024-11-10T22:39:58,711 INFO [StoreOpener-74244978b603a65b5f2074c624cb7957-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 74244978b603a65b5f2074c624cb7957 2024-11-10T22:39:58,712 INFO [StoreOpener-74244978b603a65b5f2074c624cb7957-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 74244978b603a65b5f2074c624cb7957 columnFamilyName A 2024-11-10T22:39:58,712 DEBUG [StoreOpener-74244978b603a65b5f2074c624cb7957-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:58,712 INFO [StoreOpener-74244978b603a65b5f2074c624cb7957-1 {}] regionserver.HStore(327): Store=74244978b603a65b5f2074c624cb7957/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T22:39:58,713 INFO [StoreOpener-74244978b603a65b5f2074c624cb7957-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 74244978b603a65b5f2074c624cb7957 2024-11-10T22:39:58,713 INFO [StoreOpener-74244978b603a65b5f2074c624cb7957-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 74244978b603a65b5f2074c624cb7957 columnFamilyName B 2024-11-10T22:39:58,713 DEBUG [StoreOpener-74244978b603a65b5f2074c624cb7957-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:58,714 INFO [StoreOpener-74244978b603a65b5f2074c624cb7957-1 {}] regionserver.HStore(327): Store=74244978b603a65b5f2074c624cb7957/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T22:39:58,714 INFO [StoreOpener-74244978b603a65b5f2074c624cb7957-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 74244978b603a65b5f2074c624cb7957 2024-11-10T22:39:58,715 INFO [StoreOpener-74244978b603a65b5f2074c624cb7957-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 74244978b603a65b5f2074c624cb7957 columnFamilyName C 2024-11-10T22:39:58,715 DEBUG [StoreOpener-74244978b603a65b5f2074c624cb7957-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:39:58,715 INFO [StoreOpener-74244978b603a65b5f2074c624cb7957-1 {}] regionserver.HStore(327): Store=74244978b603a65b5f2074c624cb7957/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T22:39:58,715 DEBUG [RS_OPEN_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.HRegion(1038): replaying wal for 74244978b603a65b5f2074c624cb7957 2024-11-10T22:39:58,715 DEBUG [RS_OPEN_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957 2024-11-10T22:39:58,716 DEBUG [RS_OPEN_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957 2024-11-10T22:39:58,716 DEBUG [RS_OPEN_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.HRegion(1048): stopping wal replay for 74244978b603a65b5f2074c624cb7957 2024-11-10T22:39:58,716 DEBUG [RS_OPEN_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.HRegion(1060): Cleaning up temporary data for 74244978b603a65b5f2074c624cb7957 2024-11-10T22:39:58,717 DEBUG [RS_OPEN_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-10T22:39:58,718 DEBUG [RS_OPEN_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.HRegion(1093): writing seq id for 74244978b603a65b5f2074c624cb7957 2024-11-10T22:39:58,719 DEBUG [RS_OPEN_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-10T22:39:58,720 INFO [RS_OPEN_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.HRegion(1114): Opened 74244978b603a65b5f2074c624cb7957; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70830378, jitterRate=0.05545487999916077}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-10T22:39:58,720 DEBUG [RS_OPEN_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 74244978b603a65b5f2074c624cb7957 2024-11-10T22:39:58,720 DEBUG [RS_OPEN_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.HRegion(1006): Region open journal for 74244978b603a65b5f2074c624cb7957: Running coprocessor pre-open hook at 1731278398710Writing region info on filesystem at 1731278398710Initializing all the Stores at 1731278398710Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731278398710Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731278398711 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731278398711Cleaning up temporary data from old regions at 1731278398716 (+5 ms)Running coprocessor post-open hooks at 1731278398720 (+4 ms)Region opened successfully at 1731278398720 2024-11-10T22:39:58,721 INFO [RS_OPEN_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., pid=154, masterSystemTime=1731278398707 2024-11-10T22:39:58,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-10T22:39:58,722 DEBUG [RS_OPEN_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. 2024-11-10T22:39:58,722 INFO [RS_OPEN_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. 2024-11-10T22:39:58,723 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=153 updating hbase:meta row=74244978b603a65b5f2074c624cb7957, regionState=OPEN, openSeqNum=2, regionLocation=849d0d71310f,43753,1731278250663 2024-11-10T22:39:58,724 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=154, ppid=153, state=RUNNABLE, hasLock=false; OpenRegionProcedure 74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 because future has completed 2024-11-10T22:39:58,726 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=154, resume processing ppid=153 2024-11-10T22:39:58,726 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, ppid=153, state=SUCCESS, hasLock=false; OpenRegionProcedure 74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 in 170 msec 2024-11-10T22:39:58,727 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=152 2024-11-10T22:39:58,727 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=152, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=74244978b603a65b5f2074c624cb7957, ASSIGN in 325 msec 2024-11-10T22:39:58,728 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=152, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-10T22:39:58,728 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731278398728"}]},"ts":"1731278398728"} 2024-11-10T22:39:58,729 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-10T22:39:58,729 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=152, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-10T22:39:58,730 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1460 sec 2024-11-10T22:39:59,064 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-10T22:39:59,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-10T22:39:59,732 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-10T22:39:59,733 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a04de68, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:39:59,733 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849d0d71310f,43351,-1 for getting cluster id 2024-11-10T22:39:59,733 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T22:39:59,734 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59ccc382-681c-4202-a6b6-e337b1afc6c7' 2024-11-10T22:39:59,734 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T22:39:59,734 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59ccc382-681c-4202-a6b6-e337b1afc6c7" 2024-11-10T22:39:59,735 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4321ccf5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:39:59,735 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849d0d71310f,43351,-1] 2024-11-10T22:39:59,735 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T22:39:59,735 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:39:59,736 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53724, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T22:39:59,736 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ea0881, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:39:59,737 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T22:39:59,737 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849d0d71310f,43753,1731278250663, seqNum=-1] 2024-11-10T22:39:59,738 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T22:39:59,738 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34508, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T22:39:59,739 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-10T22:39:59,740 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is 849d0d71310f,43351,1731278249979 2024-11-10T22:39:59,740 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@6c266d8d 2024-11-10T22:39:59,740 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-10T22:39:59,740 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53728, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-10T22:39:59,742 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-10T22:39:59,742 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-10T22:39:59,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] procedure2.ProcedureExecutor(1139): Stored pid=155, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-10T22:39:59,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742349_1525 (size=995) 2024-11-10T22:40:00,152 DEBUG [PEWorker-3 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-10T22:40:00,152 INFO [PEWorker-3 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-10T22:40:00,154 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=156, ppid=155, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-10T22:40:00,156 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=74244978b603a65b5f2074c624cb7957, REOPEN/MOVE}] 2024-11-10T22:40:00,156 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=157, ppid=156, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=74244978b603a65b5f2074c624cb7957, REOPEN/MOVE 2024-11-10T22:40:00,157 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=157 updating hbase:meta row=74244978b603a65b5f2074c624cb7957, regionState=CLOSING, regionLocation=849d0d71310f,43753,1731278250663 2024-11-10T22:40:00,158 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=157, ppid=156, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=74244978b603a65b5f2074c624cb7957, REOPEN/MOVE because future has completed 2024-11-10T22:40:00,158 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-10T22:40:00,158 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=158, ppid=157, state=RUNNABLE, hasLock=false; CloseRegionProcedure 74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663}] 2024-11-10T22:40:00,311 INFO [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=158}] handler.UnassignRegionHandler(122): Close 74244978b603a65b5f2074c624cb7957 2024-11-10T22:40:00,311 DEBUG [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=158}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-10T22:40:00,311 DEBUG [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=158}] regionserver.HRegion(1722): Closing 74244978b603a65b5f2074c624cb7957, disabling compactions & flushes 2024-11-10T22:40:00,311 INFO [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=158}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. 2024-11-10T22:40:00,311 DEBUG [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=158}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. 2024-11-10T22:40:00,311 DEBUG [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=158}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. after waiting 0 ms 2024-11-10T22:40:00,311 DEBUG [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=158}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. 2024-11-10T22:40:00,314 DEBUG [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=158}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-10T22:40:00,314 INFO [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=158}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. 2024-11-10T22:40:00,314 DEBUG [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=158}] regionserver.HRegion(1676): Region close journal for 74244978b603a65b5f2074c624cb7957: Waiting for close lock at 1731278400311Running coprocessor pre-close hooks at 1731278400311Disabling compacts and flushes for region at 1731278400311Disabling writes for close at 1731278400311Writing region close event to WAL at 1731278400311Running coprocessor post-close hooks at 1731278400314 (+3 ms)Closed at 1731278400314 2024-11-10T22:40:00,315 WARN [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=158}] regionserver.HRegionServer(3299): Not adding moved region record: 74244978b603a65b5f2074c624cb7957 to self. 2024-11-10T22:40:00,316 INFO [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=158}] handler.UnassignRegionHandler(157): Closed 74244978b603a65b5f2074c624cb7957 2024-11-10T22:40:00,316 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=157 updating hbase:meta row=74244978b603a65b5f2074c624cb7957, regionState=CLOSED 2024-11-10T22:40:00,318 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=158, ppid=157, state=RUNNABLE, hasLock=false; CloseRegionProcedure 74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 because future has completed 2024-11-10T22:40:00,319 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=158, resume processing ppid=157 2024-11-10T22:40:00,319 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, ppid=157, state=SUCCESS, hasLock=false; CloseRegionProcedure 74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 in 160 msec 2024-11-10T22:40:00,320 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=157, ppid=156, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=74244978b603a65b5f2074c624cb7957, REOPEN/MOVE; state=CLOSED, location=849d0d71310f,43753,1731278250663; forceNewPlan=false, retain=true 2024-11-10T22:40:00,392 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-10T22:40:00,471 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=157 updating hbase:meta row=74244978b603a65b5f2074c624cb7957, regionState=OPENING, regionLocation=849d0d71310f,43753,1731278250663 2024-11-10T22:40:00,472 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=157, ppid=156, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=74244978b603a65b5f2074c624cb7957, REOPEN/MOVE because future has completed 2024-11-10T22:40:00,472 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=157, state=RUNNABLE, hasLock=false; OpenRegionProcedure 74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663}] 2024-11-10T22:40:00,626 INFO [RS_OPEN_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. 2024-11-10T22:40:00,626 DEBUG [RS_OPEN_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] regionserver.HRegion(7752): Opening region: {ENCODED => 74244978b603a65b5f2074c624cb7957, NAME => 'TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.', STARTKEY => '', ENDKEY => ''} 2024-11-10T22:40:00,626 DEBUG [RS_OPEN_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 74244978b603a65b5f2074c624cb7957 2024-11-10T22:40:00,627 DEBUG [RS_OPEN_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T22:40:00,627 DEBUG [RS_OPEN_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] regionserver.HRegion(7794): checking encryption for 74244978b603a65b5f2074c624cb7957 2024-11-10T22:40:00,627 DEBUG [RS_OPEN_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] regionserver.HRegion(7797): checking classloading for 74244978b603a65b5f2074c624cb7957 2024-11-10T22:40:00,628 INFO [StoreOpener-74244978b603a65b5f2074c624cb7957-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 74244978b603a65b5f2074c624cb7957 2024-11-10T22:40:00,628 INFO [StoreOpener-74244978b603a65b5f2074c624cb7957-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 74244978b603a65b5f2074c624cb7957 columnFamilyName A 2024-11-10T22:40:00,629 DEBUG [StoreOpener-74244978b603a65b5f2074c624cb7957-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:00,630 INFO [StoreOpener-74244978b603a65b5f2074c624cb7957-1 {}] regionserver.HStore(327): Store=74244978b603a65b5f2074c624cb7957/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T22:40:00,630 INFO [StoreOpener-74244978b603a65b5f2074c624cb7957-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 74244978b603a65b5f2074c624cb7957 2024-11-10T22:40:00,630 INFO [StoreOpener-74244978b603a65b5f2074c624cb7957-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 74244978b603a65b5f2074c624cb7957 columnFamilyName B 2024-11-10T22:40:00,630 DEBUG [StoreOpener-74244978b603a65b5f2074c624cb7957-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:00,631 INFO [StoreOpener-74244978b603a65b5f2074c624cb7957-1 {}] regionserver.HStore(327): Store=74244978b603a65b5f2074c624cb7957/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T22:40:00,631 INFO [StoreOpener-74244978b603a65b5f2074c624cb7957-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 74244978b603a65b5f2074c624cb7957 2024-11-10T22:40:00,631 INFO [StoreOpener-74244978b603a65b5f2074c624cb7957-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 74244978b603a65b5f2074c624cb7957 columnFamilyName C 2024-11-10T22:40:00,631 DEBUG [StoreOpener-74244978b603a65b5f2074c624cb7957-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:00,631 INFO [StoreOpener-74244978b603a65b5f2074c624cb7957-1 {}] regionserver.HStore(327): Store=74244978b603a65b5f2074c624cb7957/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T22:40:00,632 DEBUG [RS_OPEN_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] regionserver.HRegion(1038): replaying wal for 74244978b603a65b5f2074c624cb7957 2024-11-10T22:40:00,632 DEBUG [RS_OPEN_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957 2024-11-10T22:40:00,633 DEBUG [RS_OPEN_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957 2024-11-10T22:40:00,633 DEBUG [RS_OPEN_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] regionserver.HRegion(1048): stopping wal replay for 74244978b603a65b5f2074c624cb7957 2024-11-10T22:40:00,633 DEBUG [RS_OPEN_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] regionserver.HRegion(1060): Cleaning up temporary data for 74244978b603a65b5f2074c624cb7957 2024-11-10T22:40:00,634 DEBUG [RS_OPEN_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-10T22:40:00,635 DEBUG [RS_OPEN_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] regionserver.HRegion(1093): writing seq id for 74244978b603a65b5f2074c624cb7957 2024-11-10T22:40:00,635 INFO [RS_OPEN_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] regionserver.HRegion(1114): Opened 74244978b603a65b5f2074c624cb7957; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71082202, jitterRate=0.05920735001564026}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-10T22:40:00,635 DEBUG [RS_OPEN_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 74244978b603a65b5f2074c624cb7957 2024-11-10T22:40:00,636 DEBUG [RS_OPEN_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] regionserver.HRegion(1006): Region open journal for 74244978b603a65b5f2074c624cb7957: Running coprocessor pre-open hook at 1731278400627Writing region info on filesystem at 1731278400627Initializing all the Stores at 1731278400627Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731278400627Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731278400627Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731278400627Cleaning up temporary data from old regions at 1731278400633 (+6 ms)Running coprocessor post-open hooks at 1731278400635 (+2 ms)Region opened successfully at 1731278400636 (+1 ms) 2024-11-10T22:40:00,636 INFO [RS_OPEN_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., pid=159, masterSystemTime=1731278400624 2024-11-10T22:40:00,638 DEBUG [RS_OPEN_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. 2024-11-10T22:40:00,638 INFO [RS_OPEN_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. 2024-11-10T22:40:00,638 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=157 updating hbase:meta row=74244978b603a65b5f2074c624cb7957, regionState=OPEN, openSeqNum=5, regionLocation=849d0d71310f,43753,1731278250663 2024-11-10T22:40:00,640 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=159, ppid=157, state=RUNNABLE, hasLock=false; OpenRegionProcedure 74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 because future has completed 2024-11-10T22:40:00,641 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=157 2024-11-10T22:40:00,641 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=157, state=SUCCESS, hasLock=false; OpenRegionProcedure 74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 in 168 msec 2024-11-10T22:40:00,643 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-11-10T22:40:00,643 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=74244978b603a65b5f2074c624cb7957, REOPEN/MOVE in 485 msec 2024-11-10T22:40:00,645 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=156, resume processing ppid=155 2024-11-10T22:40:00,645 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, ppid=155, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 489 msec 2024-11-10T22:40:00,646 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 903 msec 2024-11-10T22:40:00,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=155 2024-11-10T22:40:00,647 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-10T22:40:00,648 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53954261, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:40:00,648 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849d0d71310f,43351,-1 for getting cluster id 2024-11-10T22:40:00,648 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T22:40:00,649 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59ccc382-681c-4202-a6b6-e337b1afc6c7' 2024-11-10T22:40:00,649 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T22:40:00,649 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59ccc382-681c-4202-a6b6-e337b1afc6c7" 2024-11-10T22:40:00,649 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5803d6c7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:40:00,649 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849d0d71310f,43351,-1] 2024-11-10T22:40:00,649 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T22:40:00,649 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:40:00,650 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53760, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T22:40:00,650 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5699c1c9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:40:00,651 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e2aa61c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:40:00,651 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849d0d71310f,43351,-1 for getting cluster id 2024-11-10T22:40:00,651 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T22:40:00,652 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59ccc382-681c-4202-a6b6-e337b1afc6c7' 2024-11-10T22:40:00,652 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T22:40:00,652 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59ccc382-681c-4202-a6b6-e337b1afc6c7" 2024-11-10T22:40:00,652 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27c19d53, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:40:00,652 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849d0d71310f,43351,-1] 2024-11-10T22:40:00,652 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T22:40:00,652 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:40:00,653 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53782, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T22:40:00,653 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c9ccbbf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:40:00,654 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ca6c80d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:40:00,654 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849d0d71310f,43351,-1 for getting cluster id 2024-11-10T22:40:00,654 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T22:40:00,654 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59ccc382-681c-4202-a6b6-e337b1afc6c7' 2024-11-10T22:40:00,655 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T22:40:00,655 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59ccc382-681c-4202-a6b6-e337b1afc6c7" 2024-11-10T22:40:00,655 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5dd1cff4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:40:00,655 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849d0d71310f,43351,-1] 2024-11-10T22:40:00,655 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T22:40:00,655 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:40:00,656 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53808, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T22:40:00,656 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d06c98e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:40:00,657 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14afc90b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:40:00,657 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849d0d71310f,43351,-1 for getting cluster id 2024-11-10T22:40:00,657 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T22:40:00,657 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59ccc382-681c-4202-a6b6-e337b1afc6c7' 2024-11-10T22:40:00,657 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T22:40:00,657 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59ccc382-681c-4202-a6b6-e337b1afc6c7" 2024-11-10T22:40:00,658 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4df1fbb2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:40:00,658 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849d0d71310f,43351,-1] 2024-11-10T22:40:00,658 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T22:40:00,658 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:40:00,658 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53824, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T22:40:00,659 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b741874, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:40:00,659 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1cb57557, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:40:00,659 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849d0d71310f,43351,-1 for getting cluster id 2024-11-10T22:40:00,659 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T22:40:00,660 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59ccc382-681c-4202-a6b6-e337b1afc6c7' 2024-11-10T22:40:00,660 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T22:40:00,660 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59ccc382-681c-4202-a6b6-e337b1afc6c7" 2024-11-10T22:40:00,660 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d246e35, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:40:00,660 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849d0d71310f,43351,-1] 2024-11-10T22:40:00,660 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T22:40:00,660 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:40:00,661 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53840, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T22:40:00,661 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@340a209b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:40:00,662 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@99b73f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:40:00,662 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849d0d71310f,43351,-1 for getting cluster id 2024-11-10T22:40:00,662 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T22:40:00,662 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59ccc382-681c-4202-a6b6-e337b1afc6c7' 2024-11-10T22:40:00,662 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T22:40:00,663 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59ccc382-681c-4202-a6b6-e337b1afc6c7" 2024-11-10T22:40:00,663 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58ac7a1e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:40:00,663 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849d0d71310f,43351,-1] 2024-11-10T22:40:00,663 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T22:40:00,663 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:40:00,663 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53866, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T22:40:00,664 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15b1fe4d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:40:00,664 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@af69a6d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:40:00,664 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849d0d71310f,43351,-1 for getting cluster id 2024-11-10T22:40:00,664 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T22:40:00,665 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59ccc382-681c-4202-a6b6-e337b1afc6c7' 2024-11-10T22:40:00,665 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T22:40:00,665 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59ccc382-681c-4202-a6b6-e337b1afc6c7" 2024-11-10T22:40:00,665 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@da55e25, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:40:00,665 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849d0d71310f,43351,-1] 2024-11-10T22:40:00,665 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T22:40:00,665 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:40:00,666 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53884, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T22:40:00,666 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69f208b3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:40:00,667 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4fd90259, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:40:00,667 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849d0d71310f,43351,-1 for getting cluster id 2024-11-10T22:40:00,667 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T22:40:00,668 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59ccc382-681c-4202-a6b6-e337b1afc6c7' 2024-11-10T22:40:00,668 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T22:40:00,668 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59ccc382-681c-4202-a6b6-e337b1afc6c7" 2024-11-10T22:40:00,668 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24c65041, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:40:00,668 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849d0d71310f,43351,-1] 2024-11-10T22:40:00,668 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T22:40:00,668 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:40:00,669 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53906, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T22:40:00,669 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c727129, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:40:00,670 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@594d5ccc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:40:00,670 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849d0d71310f,43351,-1 for getting cluster id 2024-11-10T22:40:00,670 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T22:40:00,671 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59ccc382-681c-4202-a6b6-e337b1afc6c7' 2024-11-10T22:40:00,671 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T22:40:00,671 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59ccc382-681c-4202-a6b6-e337b1afc6c7" 2024-11-10T22:40:00,671 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ce0beda, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:40:00,671 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849d0d71310f,43351,-1] 2024-11-10T22:40:00,671 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T22:40:00,671 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:40:00,672 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53924, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T22:40:00,672 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4fb9f831, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:40:00,673 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@267cfa21, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:40:00,673 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849d0d71310f,43351,-1 for getting cluster id 2024-11-10T22:40:00,673 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T22:40:00,673 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59ccc382-681c-4202-a6b6-e337b1afc6c7' 2024-11-10T22:40:00,673 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T22:40:00,673 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59ccc382-681c-4202-a6b6-e337b1afc6c7" 2024-11-10T22:40:00,673 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@168eecf5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:40:00,673 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849d0d71310f,43351,-1] 2024-11-10T22:40:00,674 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T22:40:00,674 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:40:00,674 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53934, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T22:40:00,674 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f919a1b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T22:40:00,675 DEBUG [Thread-2365 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T22:40:00,675 DEBUG [Thread-2361 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T22:40:00,675 DEBUG [Thread-2359 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T22:40:00,675 DEBUG [Thread-2350 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T22:40:00,675 DEBUG [Thread-2354 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T22:40:00,675 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T22:40:00,676 DEBUG [Thread-2363 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T22:40:00,676 DEBUG [Thread-2346 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T22:40:00,676 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849d0d71310f,43753,1731278250663, seqNum=-1] 2024-11-10T22:40:00,676 DEBUG [Thread-2357 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T22:40:00,676 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849d0d71310f,43753,1731278250663, seqNum=-1] 2024-11-10T22:40:00,676 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849d0d71310f,43753,1731278250663, seqNum=-1] 2024-11-10T22:40:00,676 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849d0d71310f,43753,1731278250663, seqNum=-1] 2024-11-10T22:40:00,676 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T22:40:00,676 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T22:40:00,676 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T22:40:00,676 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T22:40:00,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43351 {}] procedure2.ProcedureExecutor(1139): Stored pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees 2024-11-10T22:40:00,677 DEBUG [Thread-2348 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T22:40:00,677 DEBUG [Thread-2357 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849d0d71310f,43753,1731278250663, seqNum=-1] 2024-11-10T22:40:00,677 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34512, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T22:40:00,677 DEBUG [Thread-2357 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T22:40:00,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-10T22:40:00,677 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34520, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T22:40:00,678 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34536, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T22:40:00,678 DEBUG [Thread-2352 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T22:40:00,678 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34540, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T22:40:00,678 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849d0d71310f,43753,1731278250663, seqNum=-1] 2024-11-10T22:40:00,678 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T22:40:00,678 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T22:40:00,678 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849d0d71310f,43753,1731278250663, seqNum=-1] 2024-11-10T22:40:00,678 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849d0d71310f,43753,1731278250663, seqNum=-1] 2024-11-10T22:40:00,679 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T22:40:00,679 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T22:40:00,679 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849d0d71310f,43753,1731278250663, seqNum=-1] 2024-11-10T22:40:00,679 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T22:40:00,679 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T22:40:00,679 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T22:40:00,679 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34558, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T22:40:00,679 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34566, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T22:40:00,680 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34580, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T22:40:00,680 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5] 2024-11-10T22:40:00,680 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5] 2024-11-10T22:40:00,681 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5] 2024-11-10T22:40:00,681 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34544, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T22:40:00,681 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34576, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T22:40:00,682 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5] 2024-11-10T22:40:00,682 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5] 2024-11-10T22:40:00,683 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5] 2024-11-10T22:40:00,683 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5] 2024-11-10T22:40:00,684 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849d0d71310f,43753,1731278250663, seqNum=-1] 2024-11-10T22:40:00,684 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T22:40:00,684 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34584, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T22:40:00,685 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5] 2024-11-10T22:40:00,685 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5] 2024-11-10T22:40:00,686 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5] 2024-11-10T22:40:00,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(8855): Flush requested on 74244978b603a65b5f2074c624cb7957 2024-11-10T22:40:00,692 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T22:40:00,692 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 74244978b603a65b5f2074c624cb7957 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-10T22:40:00,715 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:00,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34536 deadline: 1731278460711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:00,716 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:00,716 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:00,716 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:00,716 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411107352bf8ea63e4344a04b583d681600c7_74244978b603a65b5f2074c624cb7957 is 50, key is test_row_0/A:col10/1731278400691/Put/seqid=0 2024-11-10T22:40:00,718 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:00,718 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:00,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34580 deadline: 1731278460713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:00,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34584 deadline: 1731278460714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:00,718 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:00,719 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:00,719 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:00,719 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:00,719 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:00,719 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:00,719 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:00,719 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:00,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34566 deadline: 1731278460716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:00,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34576 deadline: 1731278460715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:00,720 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:00,720 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:00,720 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:00,720 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:00,720 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:00,720 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:00,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742350_1526 (size=12154) 2024-11-10T22:40:00,726 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:00,729 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411107352bf8ea63e4344a04b583d681600c7_74244978b603a65b5f2074c624cb7957 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411107352bf8ea63e4344a04b583d681600c7_74244978b603a65b5f2074c624cb7957 2024-11-10T22:40:00,730 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/A/bbe3f0bfc4024a6187077b135f1a8e0c, store: [table=TestAcidGuarantees family=A region=74244978b603a65b5f2074c624cb7957] 2024-11-10T22:40:00,730 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/A/bbe3f0bfc4024a6187077b135f1a8e0c is 175, key is test_row_0/A:col10/1731278400691/Put/seqid=0 2024-11-10T22:40:00,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742351_1527 (size=30955) 2024-11-10T22:40:00,734 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=17, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/A/bbe3f0bfc4024a6187077b135f1a8e0c 2024-11-10T22:40:00,758 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/B/b2c5fd95eaf0403290c6cc9cbb96e8e1 is 50, key is test_row_0/B:col10/1731278400691/Put/seqid=0 2024-11-10T22:40:00,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742352_1528 (size=12001) 2024-11-10T22:40:00,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-10T22:40:00,825 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:00,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34536 deadline: 1731278460822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:00,825 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:00,825 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:00,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34580 deadline: 1731278460822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:00,825 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:00,825 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:00,825 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:00,825 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:00,826 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:00,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34584 deadline: 1731278460823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:00,826 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:00,826 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:00,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34566 deadline: 1731278460823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:00,826 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:00,826 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:00,826 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:00,826 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:00,826 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:00,826 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:00,826 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:00,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34576 deadline: 1731278460823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:00,826 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:00,826 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:00,826 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:00,833 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-10T22:40:00,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. 2024-11-10T22:40:00,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. as already flushing 2024-11-10T22:40:00,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. 2024-11-10T22:40:00,834 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 74244978b603a65b5f2074c624cb7957, NAME => 'TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:40:00,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 74244978b603a65b5f2074c624cb7957, NAME => 'TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:40:00,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 74244978b603a65b5f2074c624cb7957, NAME => 'TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 74244978b603a65b5f2074c624cb7957, NAME => 'TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:40:00,986 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-10T22:40:00,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. 2024-11-10T22:40:00,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. as already flushing 2024-11-10T22:40:00,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. 2024-11-10T22:40:00,987 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 74244978b603a65b5f2074c624cb7957, NAME => 'TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:40:00,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 74244978b603a65b5f2074c624cb7957, NAME => 'TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:40:00,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 74244978b603a65b5f2074c624cb7957, NAME => 'TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 74244978b603a65b5f2074c624cb7957, NAME => 'TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:40:00,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-10T22:40:01,035 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:01,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34536 deadline: 1731278461032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:01,035 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:01,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34580 deadline: 1731278461032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:01,036 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:01,036 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:01,036 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:01,036 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:01,036 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:01,036 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:01,036 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:01,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34566 deadline: 1731278461032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:01,036 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:01,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34584 deadline: 1731278461032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:01,036 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:01,036 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:01,036 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:01,036 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:01,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34576 deadline: 1731278461032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:01,036 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:01,036 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:01,036 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:01,036 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:01,036 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:01,036 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:01,140 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-10T22:40:01,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. 2024-11-10T22:40:01,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. as already flushing 2024-11-10T22:40:01,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. 2024-11-10T22:40:01,140 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 74244978b603a65b5f2074c624cb7957, NAME => 'TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:40:01,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 74244978b603a65b5f2074c624cb7957, NAME => 'TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:40:01,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 74244978b603a65b5f2074c624cb7957, NAME => 'TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 74244978b603a65b5f2074c624cb7957, NAME => 'TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:40:01,162 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/B/b2c5fd95eaf0403290c6cc9cbb96e8e1 2024-11-10T22:40:01,186 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/C/449774d5398a42fe9cea117b9dba4146 is 50, key is test_row_0/C:col10/1731278400691/Put/seqid=0 2024-11-10T22:40:01,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742353_1529 (size=12001) 2024-11-10T22:40:01,190 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/C/449774d5398a42fe9cea117b9dba4146 2024-11-10T22:40:01,194 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/A/bbe3f0bfc4024a6187077b135f1a8e0c as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/A/bbe3f0bfc4024a6187077b135f1a8e0c 2024-11-10T22:40:01,197 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/A/bbe3f0bfc4024a6187077b135f1a8e0c, entries=150, sequenceid=17, filesize=30.2 K 2024-11-10T22:40:01,197 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/B/b2c5fd95eaf0403290c6cc9cbb96e8e1 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/B/b2c5fd95eaf0403290c6cc9cbb96e8e1 2024-11-10T22:40:01,200 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/B/b2c5fd95eaf0403290c6cc9cbb96e8e1, entries=150, sequenceid=17, filesize=11.7 K 2024-11-10T22:40:01,200 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/C/449774d5398a42fe9cea117b9dba4146 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/C/449774d5398a42fe9cea117b9dba4146 2024-11-10T22:40:01,203 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/C/449774d5398a42fe9cea117b9dba4146, entries=150, sequenceid=17, filesize=11.7 K 2024-11-10T22:40:01,204 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 74244978b603a65b5f2074c624cb7957 in 512ms, sequenceid=17, compaction requested=false 2024-11-10T22:40:01,205 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-10T22:40:01,205 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 74244978b603a65b5f2074c624cb7957: 2024-11-10T22:40:01,293 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-10T22:40:01,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. 2024-11-10T22:40:01,293 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2902): Flushing 74244978b603a65b5f2074c624cb7957 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-10T22:40:01,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110486b44ec56a7403899edefe20ec943b1_74244978b603a65b5f2074c624cb7957 is 50, key is test_row_0/A:col10/1731278400712/Put/seqid=0 2024-11-10T22:40:01,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-10T22:40:01,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742354_1530 (size=12154) 2024-11-10T22:40:01,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:01,315 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110486b44ec56a7403899edefe20ec943b1_74244978b603a65b5f2074c624cb7957 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110486b44ec56a7403899edefe20ec943b1_74244978b603a65b5f2074c624cb7957 2024-11-10T22:40:01,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/A/aeff6581916c45eca3665e7cc3ed057f, store: [table=TestAcidGuarantees family=A region=74244978b603a65b5f2074c624cb7957] 2024-11-10T22:40:01,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/A/aeff6581916c45eca3665e7cc3ed057f is 175, key is test_row_0/A:col10/1731278400712/Put/seqid=0 2024-11-10T22:40:01,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742355_1531 (size=30955) 2024-11-10T22:40:01,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(8855): Flush requested on 74244978b603a65b5f2074c624cb7957 2024-11-10T22:40:01,344 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. as already flushing 2024-11-10T22:40:01,356 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:01,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34536 deadline: 1731278461351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:01,357 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:01,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34580 deadline: 1731278461351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:01,357 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:01,357 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:01,357 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:01,357 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:01,357 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:01,357 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:01,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34566 deadline: 1731278461352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:01,357 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:01,357 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:01,357 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:01,357 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:01,357 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:01,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34576 deadline: 1731278461354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:01,358 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:01,358 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:01,358 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:01,358 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:01,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34584 deadline: 1731278461354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:01,358 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:01,358 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:01,358 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:01,465 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:01,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34536 deadline: 1731278461462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:01,466 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:01,466 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:01,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34580 deadline: 1731278461463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:01,466 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:01,466 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:01,466 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:01,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34566 deadline: 1731278461463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:01,466 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:01,466 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:01,466 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:01,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34576 deadline: 1731278461463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:01,466 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:01,466 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:01,466 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:01,466 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:01,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34584 deadline: 1731278461463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:01,467 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:01,467 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:01,467 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:01,467 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:01,467 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:01,467 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:01,467 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:01,676 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:01,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34536 deadline: 1731278461673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:01,676 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:01,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34580 deadline: 1731278461673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:01,676 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:01,677 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:01,677 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:01,677 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:01,677 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:01,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34566 deadline: 1731278461673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:01,677 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:01,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34576 deadline: 1731278461673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:01,677 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:01,677 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:01,677 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:01,677 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:01,677 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:01,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34584 deadline: 1731278461673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:01,677 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:01,677 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:01,677 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:01,677 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:01,677 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:01,677 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:01,677 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:01,721 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=40, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/A/aeff6581916c45eca3665e7cc3ed057f 2024-11-10T22:40:01,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/B/b1e43329d5874f0ea1d8bfcd751c884e is 50, key is test_row_0/B:col10/1731278400712/Put/seqid=0 2024-11-10T22:40:01,731 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742356_1532 (size=12001) 2024-11-10T22:40:01,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-10T22:40:01,985 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:01,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34536 deadline: 1731278461983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:01,985 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:01,985 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:01,985 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:01,985 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:01,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34580 deadline: 1731278461983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:01,985 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:01,985 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:01,985 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:01,985 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:01,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34566 deadline: 1731278461983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:01,986 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:01,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34576 deadline: 1731278461983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:01,986 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:01,986 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:01,986 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:01,986 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:01,986 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:01,986 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:01,986 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:01,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34584 deadline: 1731278461983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:01,986 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:01,986 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:01,986 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:02,132 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/B/b1e43329d5874f0ea1d8bfcd751c884e 2024-11-10T22:40:02,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/C/fa9dfce9206946cd954a5aa298a1bc0c is 50, key is test_row_0/C:col10/1731278400712/Put/seqid=0 2024-11-10T22:40:02,143 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742357_1533 (size=12001) 2024-11-10T22:40:02,281 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-10T22:40:02,496 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:02,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34536 deadline: 1731278462493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:02,496 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:02,496 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:02,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34580 deadline: 1731278462493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:02,496 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:02,496 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:02,496 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:02,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34566 deadline: 1731278462493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:02,497 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:02,497 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:02,497 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:02,497 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:02,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34576 deadline: 1731278462493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:02,497 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:02,497 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:02,497 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:02,497 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:02,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34584 deadline: 1731278462493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:02,497 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:02,497 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:02,497 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:02,497 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:02,497 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:02,497 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:02,544 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/C/fa9dfce9206946cd954a5aa298a1bc0c 2024-11-10T22:40:02,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/A/aeff6581916c45eca3665e7cc3ed057f as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/A/aeff6581916c45eca3665e7cc3ed057f 2024-11-10T22:40:02,551 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/A/aeff6581916c45eca3665e7cc3ed057f, entries=150, sequenceid=40, filesize=30.2 K 2024-11-10T22:40:02,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/B/b1e43329d5874f0ea1d8bfcd751c884e as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/B/b1e43329d5874f0ea1d8bfcd751c884e 2024-11-10T22:40:02,555 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/B/b1e43329d5874f0ea1d8bfcd751c884e, entries=150, sequenceid=40, filesize=11.7 K 2024-11-10T22:40:02,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/C/fa9dfce9206946cd954a5aa298a1bc0c as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/C/fa9dfce9206946cd954a5aa298a1bc0c 2024-11-10T22:40:02,559 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/C/fa9dfce9206946cd954a5aa298a1bc0c, entries=150, sequenceid=40, filesize=11.7 K 2024-11-10T22:40:02,560 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 74244978b603a65b5f2074c624cb7957 in 1267ms, sequenceid=40, compaction requested=false 2024-11-10T22:40:02,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2603): Flush status journal for 74244978b603a65b5f2074c624cb7957: 2024-11-10T22:40:02,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. 2024-11-10T22:40:02,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=161 2024-11-10T22:40:02,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.HMaster(4169): Remote procedure done, pid=161 2024-11-10T22:40:02,564 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-11-10T22:40:02,564 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8830 sec 2024-11-10T22:40:02,566 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees in 1.8890 sec 2024-11-10T22:40:02,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-10T22:40:02,823 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T22:40:02,824 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T22:40:02,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] procedure2.ProcedureExecutor(1139): Stored pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees 2024-11-10T22:40:02,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-10T22:40:02,826 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T22:40:02,827 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T22:40:02,827 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=163, ppid=162, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T22:40:02,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-10T22:40:02,979 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-10T22:40:02,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. 2024-11-10T22:40:02,979 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2902): Flushing 74244978b603a65b5f2074c624cb7957 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-10T22:40:02,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110369aa87bd4e9446e86fdf05223ddf861_74244978b603a65b5f2074c624cb7957 is 50, key is test_row_0/A:col10/1731278401352/Put/seqid=0 2024-11-10T22:40:02,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742358_1534 (size=12154) 2024-11-10T22:40:03,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-10T22:40:03,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:03,393 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110369aa87bd4e9446e86fdf05223ddf861_74244978b603a65b5f2074c624cb7957 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110369aa87bd4e9446e86fdf05223ddf861_74244978b603a65b5f2074c624cb7957 2024-11-10T22:40:03,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/A/e68d4ca0bd184424851426c45d6574a5, store: [table=TestAcidGuarantees family=A region=74244978b603a65b5f2074c624cb7957] 2024-11-10T22:40:03,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/A/e68d4ca0bd184424851426c45d6574a5 is 175, key is test_row_0/A:col10/1731278401352/Put/seqid=0 2024-11-10T22:40:03,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742359_1535 (size=30955) 2024-11-10T22:40:03,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-10T22:40:03,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(8855): Flush requested on 74244978b603a65b5f2074c624cb7957 2024-11-10T22:40:03,504 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. as already flushing 2024-11-10T22:40:03,535 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:03,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34536 deadline: 1731278463529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:03,536 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:03,536 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:03,536 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:03,537 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:03,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34580 deadline: 1731278463533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:03,538 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:03,538 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:03,538 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:03,538 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:03,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34584 deadline: 1731278463534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:03,538 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:03,538 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:03,538 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:03,541 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:03,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34566 deadline: 1731278463535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:03,541 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:03,541 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:03,541 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:03,541 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:03,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34576 deadline: 1731278463535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:03,542 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:03,542 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:03,542 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:03,646 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:03,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34536 deadline: 1731278463643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:03,646 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:03,646 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:03,646 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:03,646 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:03,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34580 deadline: 1731278463643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:03,647 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:03,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34584 deadline: 1731278463643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:03,647 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:03,647 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:03,647 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:03,647 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:03,647 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:03,647 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:03,656 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:03,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34566 deadline: 1731278463652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:03,656 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:03,656 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:03,656 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:03,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34576 deadline: 1731278463652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:03,656 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:03,656 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:03,656 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:03,657 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:03,798 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=53, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/A/e68d4ca0bd184424851426c45d6574a5 2024-11-10T22:40:03,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/B/91ba298ae7734a59b9802df3fe5088ee is 50, key is test_row_0/B:col10/1731278401352/Put/seqid=0 2024-11-10T22:40:03,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742360_1536 (size=12001) 2024-11-10T22:40:03,809 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/B/91ba298ae7734a59b9802df3fe5088ee 2024-11-10T22:40:03,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/C/fd68d70a7314421f8bfdf96aa0828b06 is 50, key is test_row_0/C:col10/1731278401352/Put/seqid=0 2024-11-10T22:40:03,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742361_1537 (size=12001) 2024-11-10T22:40:03,823 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/C/fd68d70a7314421f8bfdf96aa0828b06 2024-11-10T22:40:03,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/A/e68d4ca0bd184424851426c45d6574a5 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/A/e68d4ca0bd184424851426c45d6574a5 2024-11-10T22:40:03,830 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/A/e68d4ca0bd184424851426c45d6574a5, entries=150, sequenceid=53, filesize=30.2 K 2024-11-10T22:40:03,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/B/91ba298ae7734a59b9802df3fe5088ee as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/B/91ba298ae7734a59b9802df3fe5088ee 2024-11-10T22:40:03,834 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/B/91ba298ae7734a59b9802df3fe5088ee, entries=150, sequenceid=53, filesize=11.7 K 2024-11-10T22:40:03,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/C/fd68d70a7314421f8bfdf96aa0828b06 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/C/fd68d70a7314421f8bfdf96aa0828b06 2024-11-10T22:40:03,838 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/C/fd68d70a7314421f8bfdf96aa0828b06, entries=150, sequenceid=53, filesize=11.7 K 2024-11-10T22:40:03,839 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=140.89 KB/144270 for 74244978b603a65b5f2074c624cb7957 in 860ms, sequenceid=53, compaction requested=true 2024-11-10T22:40:03,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2603): Flush status journal for 74244978b603a65b5f2074c624cb7957: 2024-11-10T22:40:03,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. 2024-11-10T22:40:03,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=163 2024-11-10T22:40:03,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.HMaster(4169): Remote procedure done, pid=163 2024-11-10T22:40:03,842 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=163, resume processing ppid=162 2024-11-10T22:40:03,842 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=163, ppid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0130 sec 2024-11-10T22:40:03,844 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees in 1.0190 sec 2024-11-10T22:40:03,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(8855): Flush requested on 74244978b603a65b5f2074c624cb7957 2024-11-10T22:40:03,856 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T22:40:03,856 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 74244978b603a65b5f2074c624cb7957 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-10T22:40:03,863 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110ee90dd6d4ad74b9bbb7e6481725e5cd7_74244978b603a65b5f2074c624cb7957 is 50, key is test_row_0/A:col10/1731278403855/Put/seqid=0 2024-11-10T22:40:03,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742362_1538 (size=12154) 2024-11-10T22:40:03,870 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:03,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34584 deadline: 1731278463865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:03,870 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:03,870 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:03,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34536 deadline: 1731278463866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:03,870 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:03,870 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:03,871 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:03,871 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:03,871 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:03,871 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:03,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34576 deadline: 1731278463867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:03,871 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:03,871 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:03,871 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:03,875 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:03,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34566 deadline: 1731278463870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:03,875 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:03,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34580 deadline: 1731278463871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:03,876 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:03,876 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:03,876 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:03,876 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:03,876 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:03,876 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:03,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-10T22:40:03,963 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T22:40:03,963 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T22:40:03,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] procedure2.ProcedureExecutor(1139): Stored pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees 2024-11-10T22:40:03,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-10T22:40:03,965 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T22:40:03,966 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T22:40:03,966 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=165, ppid=164, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T22:40:03,976 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:03,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34536 deadline: 1731278463972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:03,976 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:03,977 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:03,977 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:03,985 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:03,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34584 deadline: 1731278463983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:03,986 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:03,986 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:03,986 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:03,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34576 deadline: 1731278463983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:03,986 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:03,986 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:03,986 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:03,986 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:03,986 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:03,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34580 deadline: 1731278463983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:03,986 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:03,986 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:03,987 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:03,987 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:03,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34566 deadline: 1731278463983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:03,987 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:03,987 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:03,987 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:04,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-10T22:40:04,119 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-10T22:40:04,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. 2024-11-10T22:40:04,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. as already flushing 2024-11-10T22:40:04,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. 2024-11-10T22:40:04,119 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 74244978b603a65b5f2074c624cb7957, NAME => 'TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:40:04,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 74244978b603a65b5f2074c624cb7957, NAME => 'TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:40:04,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 74244978b603a65b5f2074c624cb7957, NAME => 'TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 74244978b603a65b5f2074c624cb7957, NAME => 'TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:40:04,184 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:04,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34536 deadline: 1731278464183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:04,185 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:04,185 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:04,185 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:04,198 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:04,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34584 deadline: 1731278464196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:04,198 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:04,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34580 deadline: 1731278464197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:04,198 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:04,199 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:04,199 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:04,199 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:04,199 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:04,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34566 deadline: 1731278464197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:04,199 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:04,199 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:04,199 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:04,199 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:04,199 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:04,199 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:04,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34576 deadline: 1731278464197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:04,199 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:04,199 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:04,199 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:04,269 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:04,272 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-10T22:40:04,272 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110ee90dd6d4ad74b9bbb7e6481725e5cd7_74244978b603a65b5f2074c624cb7957 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110ee90dd6d4ad74b9bbb7e6481725e5cd7_74244978b603a65b5f2074c624cb7957 2024-11-10T22:40:04,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. 2024-11-10T22:40:04,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. as already flushing 2024-11-10T22:40:04,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. 2024-11-10T22:40:04,272 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 74244978b603a65b5f2074c624cb7957, NAME => 'TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:40:04,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 74244978b603a65b5f2074c624cb7957, NAME => 'TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:40:04,273 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/A/9b9bec2a61aa494ea8350881b2ed94a4, store: [table=TestAcidGuarantees family=A region=74244978b603a65b5f2074c624cb7957] 2024-11-10T22:40:04,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 74244978b603a65b5f2074c624cb7957, NAME => 'TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 74244978b603a65b5f2074c624cb7957, NAME => 'TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:40:04,274 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/A/9b9bec2a61aa494ea8350881b2ed94a4 is 175, key is test_row_0/A:col10/1731278403855/Put/seqid=0 2024-11-10T22:40:04,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742363_1539 (size=30955) 2024-11-10T22:40:04,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-10T22:40:04,424 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-10T22:40:04,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. 2024-11-10T22:40:04,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. as already flushing 2024-11-10T22:40:04,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. 2024-11-10T22:40:04,425 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 74244978b603a65b5f2074c624cb7957, NAME => 'TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:40:04,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 74244978b603a65b5f2074c624cb7957, NAME => 'TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:40:04,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 74244978b603a65b5f2074c624cb7957, NAME => 'TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 74244978b603a65b5f2074c624cb7957, NAME => 'TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:40:04,495 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:04,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34536 deadline: 1731278464492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:04,496 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:04,496 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:04,496 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:04,506 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:04,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34584 deadline: 1731278464503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:04,506 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:04,506 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:04,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34580 deadline: 1731278464503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:04,506 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:04,506 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:04,507 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:04,507 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:04,507 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:04,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34566 deadline: 1731278464503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:04,507 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:04,507 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:04,507 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:04,507 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:04,514 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:04,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34576 deadline: 1731278464512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:04,514 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:04,515 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:04,515 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:04,578 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-10T22:40:04,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. 2024-11-10T22:40:04,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. as already flushing 2024-11-10T22:40:04,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. 2024-11-10T22:40:04,578 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 74244978b603a65b5f2074c624cb7957, NAME => 'TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:40:04,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 74244978b603a65b5f2074c624cb7957, NAME => 'TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:40:04,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 74244978b603a65b5f2074c624cb7957, NAME => 'TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 74244978b603a65b5f2074c624cb7957, NAME => 'TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:40:04,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-10T22:40:04,678 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=79, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/A/9b9bec2a61aa494ea8350881b2ed94a4 2024-11-10T22:40:04,685 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/B/8fc13086e9204f7f81d0d2355e283e60 is 50, key is test_row_0/B:col10/1731278403855/Put/seqid=0 2024-11-10T22:40:04,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742364_1540 (size=12001) 2024-11-10T22:40:04,731 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-10T22:40:04,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. 2024-11-10T22:40:04,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. as already flushing 2024-11-10T22:40:04,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. 2024-11-10T22:40:04,731 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 74244978b603a65b5f2074c624cb7957, NAME => 'TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:40:04,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 74244978b603a65b5f2074c624cb7957, NAME => 'TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:40:04,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 74244978b603a65b5f2074c624cb7957, NAME => 'TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 74244978b603a65b5f2074c624cb7957, NAME => 'TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:40:04,884 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-10T22:40:04,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. 2024-11-10T22:40:04,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. as already flushing 2024-11-10T22:40:04,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. 2024-11-10T22:40:04,884 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 74244978b603a65b5f2074c624cb7957, NAME => 'TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:40:04,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 74244978b603a65b5f2074c624cb7957, NAME => 'TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:40:04,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 74244978b603a65b5f2074c624cb7957, NAME => 'TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 74244978b603a65b5f2074c624cb7957, NAME => 'TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:40:05,004 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:05,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34536 deadline: 1731278465002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:05,005 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:05,005 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:05,005 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:05,015 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:05,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34580 deadline: 1731278465013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:05,016 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:05,016 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:05,016 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:05,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34566 deadline: 1731278465013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:05,016 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:05,016 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:05,016 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:05,016 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:05,026 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:05,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34584 deadline: 1731278465022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:05,026 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:05,026 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:05,026 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:05,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34576 deadline: 1731278465022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:05,026 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:05,027 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:05,027 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:05,027 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:05,037 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-10T22:40:05,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. 2024-11-10T22:40:05,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. as already flushing 2024-11-10T22:40:05,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. 2024-11-10T22:40:05,037 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 74244978b603a65b5f2074c624cb7957, NAME => 'TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:40:05,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 74244978b603a65b5f2074c624cb7957, NAME => 'TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:40:05,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 74244978b603a65b5f2074c624cb7957, NAME => 'TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 74244978b603a65b5f2074c624cb7957, NAME => 'TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:40:05,089 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/B/8fc13086e9204f7f81d0d2355e283e60 2024-11-10T22:40:05,096 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/C/747e3bb9244f4960a2dfb96d21e0a3b2 is 50, key is test_row_0/C:col10/1731278403855/Put/seqid=0 2024-11-10T22:40:05,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742365_1541 (size=12001) 2024-11-10T22:40:05,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-10T22:40:05,189 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-10T22:40:05,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. 2024-11-10T22:40:05,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. as already flushing 2024-11-10T22:40:05,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. 2024-11-10T22:40:05,190 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 74244978b603a65b5f2074c624cb7957, NAME => 'TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:40:05,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 74244978b603a65b5f2074c624cb7957, NAME => 'TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:40:05,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 74244978b603a65b5f2074c624cb7957, NAME => 'TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 74244978b603a65b5f2074c624cb7957, NAME => 'TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:40:05,343 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-10T22:40:05,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. 2024-11-10T22:40:05,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. as already flushing 2024-11-10T22:40:05,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. 2024-11-10T22:40:05,343 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 74244978b603a65b5f2074c624cb7957, NAME => 'TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:40:05,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 74244978b603a65b5f2074c624cb7957, NAME => 'TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:40:05,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 74244978b603a65b5f2074c624cb7957, NAME => 'TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 74244978b603a65b5f2074c624cb7957, NAME => 'TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:40:05,496 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-10T22:40:05,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. 2024-11-10T22:40:05,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. as already flushing 2024-11-10T22:40:05,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. 2024-11-10T22:40:05,496 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 74244978b603a65b5f2074c624cb7957, NAME => 'TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:40:05,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 74244978b603a65b5f2074c624cb7957, NAME => 'TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:40:05,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 74244978b603a65b5f2074c624cb7957, NAME => 'TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 74244978b603a65b5f2074c624cb7957, NAME => 'TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:40:05,500 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/C/747e3bb9244f4960a2dfb96d21e0a3b2 2024-11-10T22:40:05,503 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/A/9b9bec2a61aa494ea8350881b2ed94a4 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/A/9b9bec2a61aa494ea8350881b2ed94a4 2024-11-10T22:40:05,506 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/A/9b9bec2a61aa494ea8350881b2ed94a4, entries=150, sequenceid=79, filesize=30.2 K 2024-11-10T22:40:05,507 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/B/8fc13086e9204f7f81d0d2355e283e60 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/B/8fc13086e9204f7f81d0d2355e283e60 2024-11-10T22:40:05,510 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/B/8fc13086e9204f7f81d0d2355e283e60, entries=150, sequenceid=79, filesize=11.7 K 2024-11-10T22:40:05,510 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/C/747e3bb9244f4960a2dfb96d21e0a3b2 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/C/747e3bb9244f4960a2dfb96d21e0a3b2 2024-11-10T22:40:05,513 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/C/747e3bb9244f4960a2dfb96d21e0a3b2, entries=150, sequenceid=79, filesize=11.7 K 2024-11-10T22:40:05,514 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=53.67 KB/54960 for 74244978b603a65b5f2074c624cb7957 in 1658ms, sequenceid=79, compaction requested=true 2024-11-10T22:40:05,515 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 74244978b603a65b5f2074c624cb7957: 2024-11-10T22:40:05,515 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 74244978b603a65b5f2074c624cb7957:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T22:40:05,515 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T22:40:05,515 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T22:40:05,515 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 74244978b603a65b5f2074c624cb7957:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T22:40:05,515 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T22:40:05,515 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 74244978b603a65b5f2074c624cb7957:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T22:40:05,515 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T22:40:05,515 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T22:40:05,517 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T22:40:05,517 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 123820 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T22:40:05,517 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1541): 74244978b603a65b5f2074c624cb7957/B is initiating minor compaction (all files) 2024-11-10T22:40:05,517 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HStore(1541): 74244978b603a65b5f2074c624cb7957/A is initiating minor compaction (all files) 2024-11-10T22:40:05,517 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 74244978b603a65b5f2074c624cb7957/A in TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. 2024-11-10T22:40:05,517 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 74244978b603a65b5f2074c624cb7957/B in TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. 2024-11-10T22:40:05,517 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/B/b2c5fd95eaf0403290c6cc9cbb96e8e1, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/B/b1e43329d5874f0ea1d8bfcd751c884e, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/B/91ba298ae7734a59b9802df3fe5088ee, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/B/8fc13086e9204f7f81d0d2355e283e60] into tmpdir=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp, totalSize=46.9 K 2024-11-10T22:40:05,517 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/A/bbe3f0bfc4024a6187077b135f1a8e0c, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/A/aeff6581916c45eca3665e7cc3ed057f, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/A/e68d4ca0bd184424851426c45d6574a5, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/A/9b9bec2a61aa494ea8350881b2ed94a4] into tmpdir=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp, totalSize=120.9 K 2024-11-10T22:40:05,517 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. 2024-11-10T22:40:05,517 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. files: [hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/A/bbe3f0bfc4024a6187077b135f1a8e0c, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/A/aeff6581916c45eca3665e7cc3ed057f, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/A/e68d4ca0bd184424851426c45d6574a5, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/A/9b9bec2a61aa494ea8350881b2ed94a4] 2024-11-10T22:40:05,517 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting b2c5fd95eaf0403290c6cc9cbb96e8e1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1731278400690 2024-11-10T22:40:05,517 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting bbe3f0bfc4024a6187077b135f1a8e0c, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1731278400690 2024-11-10T22:40:05,518 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting b1e43329d5874f0ea1d8bfcd751c884e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731278400710 2024-11-10T22:40:05,518 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting aeff6581916c45eca3665e7cc3ed057f, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731278400710 2024-11-10T22:40:05,518 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting e68d4ca0bd184424851426c45d6574a5, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731278401351 2024-11-10T22:40:05,518 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting 91ba298ae7734a59b9802df3fe5088ee, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731278401351 2024-11-10T22:40:05,518 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9b9bec2a61aa494ea8350881b2ed94a4, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1731278403525 2024-11-10T22:40:05,518 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting 8fc13086e9204f7f81d0d2355e283e60, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1731278403525 2024-11-10T22:40:05,524 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=74244978b603a65b5f2074c624cb7957] 2024-11-10T22:40:05,526 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 74244978b603a65b5f2074c624cb7957#B#compaction#455 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T22:40:05,526 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411107eab51ed829f4191b059cd83c38ee474_74244978b603a65b5f2074c624cb7957 store=[table=TestAcidGuarantees family=A region=74244978b603a65b5f2074c624cb7957] 2024-11-10T22:40:05,527 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/B/0f14e05a94c149d897979563f931d484 is 50, key is test_row_0/B:col10/1731278403855/Put/seqid=0 2024-11-10T22:40:05,529 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411107eab51ed829f4191b059cd83c38ee474_74244978b603a65b5f2074c624cb7957, store=[table=TestAcidGuarantees family=A region=74244978b603a65b5f2074c624cb7957] 2024-11-10T22:40:05,529 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411107eab51ed829f4191b059cd83c38ee474_74244978b603a65b5f2074c624cb7957 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=74244978b603a65b5f2074c624cb7957] 2024-11-10T22:40:05,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742366_1542 (size=12139) 2024-11-10T22:40:05,532 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742367_1543 (size=4469) 2024-11-10T22:40:05,533 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 74244978b603a65b5f2074c624cb7957#A#compaction#456 average throughput is 2.71 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T22:40:05,533 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/A/313baba61c5849e99603982923b0e38b is 175, key is test_row_0/A:col10/1731278403855/Put/seqid=0 2024-11-10T22:40:05,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742368_1544 (size=31093) 2024-11-10T22:40:05,540 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/A/313baba61c5849e99603982923b0e38b as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/A/313baba61c5849e99603982923b0e38b 2024-11-10T22:40:05,545 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 74244978b603a65b5f2074c624cb7957/A of 74244978b603a65b5f2074c624cb7957 into 313baba61c5849e99603982923b0e38b(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T22:40:05,545 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 74244978b603a65b5f2074c624cb7957: 2024-11-10T22:40:05,545 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., storeName=74244978b603a65b5f2074c624cb7957/A, priority=12, startTime=1731278405515; duration=0sec 2024-11-10T22:40:05,545 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T22:40:05,545 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 74244978b603a65b5f2074c624cb7957:A 2024-11-10T22:40:05,545 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T22:40:05,546 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T22:40:05,546 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HStore(1541): 74244978b603a65b5f2074c624cb7957/C is initiating minor compaction (all files) 2024-11-10T22:40:05,546 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 74244978b603a65b5f2074c624cb7957/C in TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. 2024-11-10T22:40:05,546 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/C/449774d5398a42fe9cea117b9dba4146, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/C/fa9dfce9206946cd954a5aa298a1bc0c, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/C/fd68d70a7314421f8bfdf96aa0828b06, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/C/747e3bb9244f4960a2dfb96d21e0a3b2] into tmpdir=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp, totalSize=46.9 K 2024-11-10T22:40:05,546 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 449774d5398a42fe9cea117b9dba4146, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1731278400690 2024-11-10T22:40:05,547 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting fa9dfce9206946cd954a5aa298a1bc0c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731278400710 2024-11-10T22:40:05,547 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting fd68d70a7314421f8bfdf96aa0828b06, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731278401351 2024-11-10T22:40:05,548 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 747e3bb9244f4960a2dfb96d21e0a3b2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1731278403525 2024-11-10T22:40:05,556 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 74244978b603a65b5f2074c624cb7957#C#compaction#457 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T22:40:05,556 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/C/bacedf3bb2194116a50d978460f44dfd is 50, key is test_row_0/C:col10/1731278403855/Put/seqid=0 2024-11-10T22:40:05,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742369_1545 (size=12139) 2024-11-10T22:40:05,572 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/C/bacedf3bb2194116a50d978460f44dfd as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/C/bacedf3bb2194116a50d978460f44dfd 2024-11-10T22:40:05,578 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 74244978b603a65b5f2074c624cb7957/C of 74244978b603a65b5f2074c624cb7957 into bacedf3bb2194116a50d978460f44dfd(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T22:40:05,578 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 74244978b603a65b5f2074c624cb7957: 2024-11-10T22:40:05,578 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., storeName=74244978b603a65b5f2074c624cb7957/C, priority=12, startTime=1731278405515; duration=0sec 2024-11-10T22:40:05,578 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T22:40:05,578 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 74244978b603a65b5f2074c624cb7957:C 2024-11-10T22:40:05,649 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-10T22:40:05,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. 2024-11-10T22:40:05,649 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2902): Flushing 74244978b603a65b5f2074c624cb7957 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-10T22:40:05,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411104bdf589343104db9b0ef37048cc75616_74244978b603a65b5f2074c624cb7957 is 50, key is test_row_0/A:col10/1731278403869/Put/seqid=0 2024-11-10T22:40:05,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742370_1546 (size=12154) 2024-11-10T22:40:05,935 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/B/0f14e05a94c149d897979563f931d484 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/B/0f14e05a94c149d897979563f931d484 2024-11-10T22:40:05,939 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 74244978b603a65b5f2074c624cb7957/B of 74244978b603a65b5f2074c624cb7957 into 0f14e05a94c149d897979563f931d484(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T22:40:05,939 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 74244978b603a65b5f2074c624cb7957: 2024-11-10T22:40:05,939 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., storeName=74244978b603a65b5f2074c624cb7957/B, priority=12, startTime=1731278405515; duration=0sec 2024-11-10T22:40:05,940 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T22:40:05,940 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 74244978b603a65b5f2074c624cb7957:B 2024-11-10T22:40:06,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(8855): Flush requested on 74244978b603a65b5f2074c624cb7957 2024-11-10T22:40:06,014 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. as already flushing 2024-11-10T22:40:06,053 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:06,053 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:06,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34580 deadline: 1731278466047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:06,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34536 deadline: 1731278466048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:06,053 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:06,053 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:06,053 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:06,053 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:06,053 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:06,053 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:06,058 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:06,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34584 deadline: 1731278466053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:06,058 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:06,058 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:06,058 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:06,058 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:06,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34566 deadline: 1731278466053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:06,059 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:06,059 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:06,059 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:06,059 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:06,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34576 deadline: 1731278466053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:06,059 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:06,059 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:06,059 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:06,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:06,077 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411104bdf589343104db9b0ef37048cc75616_74244978b603a65b5f2074c624cb7957 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411104bdf589343104db9b0ef37048cc75616_74244978b603a65b5f2074c624cb7957 2024-11-10T22:40:06,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/A/9289d78bd70d42a6901cb57c7d742a95, store: [table=TestAcidGuarantees family=A region=74244978b603a65b5f2074c624cb7957] 2024-11-10T22:40:06,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/A/9289d78bd70d42a6901cb57c7d742a95 is 175, key is test_row_0/A:col10/1731278403869/Put/seqid=0 2024-11-10T22:40:06,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742371_1547 (size=30955) 2024-11-10T22:40:06,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-10T22:40:06,164 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:06,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34536 deadline: 1731278466163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:06,164 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:06,164 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:06,164 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:06,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34580 deadline: 1731278466163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:06,164 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:06,164 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:06,164 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:06,164 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:06,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34584 deadline: 1731278466163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:06,165 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:06,165 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:06,165 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:06,165 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:06,165 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:06,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34566 deadline: 1731278466163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:06,165 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:06,165 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:06,165 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:06,165 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:06,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34576 deadline: 1731278466163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:06,165 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:06,165 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:06,165 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:06,374 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:06,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34536 deadline: 1731278466373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:06,374 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:06,374 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:06,374 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:06,374 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:06,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34580 deadline: 1731278466373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:06,374 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:06,374 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:06,374 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:06,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34584 deadline: 1731278466373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:06,374 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:06,375 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:06,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34576 deadline: 1731278466373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:06,375 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:06,375 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:06,375 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:06,375 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:06,375 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:06,375 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:06,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34566 deadline: 1731278466373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:06,375 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:06,375 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:06,375 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:06,375 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:06,482 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=92, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/A/9289d78bd70d42a6901cb57c7d742a95 2024-11-10T22:40:06,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/B/0204fa2d7c374c04a71d60ea8e01011c is 50, key is test_row_0/B:col10/1731278403869/Put/seqid=0 2024-11-10T22:40:06,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742372_1548 (size=12001) 2024-11-10T22:40:06,685 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:06,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34536 deadline: 1731278466682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:06,685 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:06,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34580 deadline: 1731278466683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:06,685 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:06,685 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:06,685 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:06,685 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:06,685 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:06,686 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:06,686 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:06,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34584 deadline: 1731278466683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:06,686 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:06,686 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:06,686 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:06,686 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:06,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34576 deadline: 1731278466683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:06,686 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:06,686 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:06,686 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:06,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34566 deadline: 1731278466683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:06,686 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:06,686 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:06,686 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:06,686 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:06,892 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/B/0204fa2d7c374c04a71d60ea8e01011c 2024-11-10T22:40:06,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/C/ca57369bdef3452ebbb4762bca3eec0b is 50, key is test_row_0/C:col10/1731278403869/Put/seqid=0 2024-11-10T22:40:06,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742373_1549 (size=12001) 2024-11-10T22:40:07,196 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:07,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34536 deadline: 1731278467193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:07,197 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:07,197 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:07,197 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:07,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34580 deadline: 1731278467193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:07,197 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:07,197 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:07,197 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:07,197 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:07,197 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:07,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34584 deadline: 1731278467193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:07,197 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:07,197 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:07,197 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:07,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34576 deadline: 1731278467193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:07,197 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:07,198 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:07,198 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:07,198 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:07,198 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:07,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34566 deadline: 1731278467193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:07,198 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:07,198 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:07,198 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:07,315 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/C/ca57369bdef3452ebbb4762bca3eec0b 2024-11-10T22:40:07,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/A/9289d78bd70d42a6901cb57c7d742a95 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/A/9289d78bd70d42a6901cb57c7d742a95 2024-11-10T22:40:07,323 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/A/9289d78bd70d42a6901cb57c7d742a95, entries=150, sequenceid=92, filesize=30.2 K 2024-11-10T22:40:07,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/B/0204fa2d7c374c04a71d60ea8e01011c as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/B/0204fa2d7c374c04a71d60ea8e01011c 2024-11-10T22:40:07,326 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/B/0204fa2d7c374c04a71d60ea8e01011c, entries=150, sequenceid=92, filesize=11.7 K 2024-11-10T22:40:07,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/C/ca57369bdef3452ebbb4762bca3eec0b as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/C/ca57369bdef3452ebbb4762bca3eec0b 2024-11-10T22:40:07,330 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/C/ca57369bdef3452ebbb4762bca3eec0b, entries=150, sequenceid=92, filesize=11.7 K 2024-11-10T22:40:07,331 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 74244978b603a65b5f2074c624cb7957 in 1682ms, sequenceid=92, compaction requested=false 2024-11-10T22:40:07,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2603): Flush status journal for 74244978b603a65b5f2074c624cb7957: 2024-11-10T22:40:07,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. 2024-11-10T22:40:07,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=165 2024-11-10T22:40:07,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.HMaster(4169): Remote procedure done, pid=165 2024-11-10T22:40:07,336 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=165, resume processing ppid=164 2024-11-10T22:40:07,336 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=165, ppid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.3680 sec 2024-11-10T22:40:07,338 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees in 3.3740 sec 2024-11-10T22:40:08,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-10T22:40:08,133 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T22:40:08,134 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T22:40:08,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] procedure2.ProcedureExecutor(1139): Stored pid=166, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees 2024-11-10T22:40:08,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-10T22:40:08,136 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=166, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T22:40:08,136 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=166, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T22:40:08,137 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=167, ppid=166, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T22:40:08,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(8855): Flush requested on 74244978b603a65b5f2074c624cb7957 2024-11-10T22:40:08,206 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T22:40:08,206 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 74244978b603a65b5f2074c624cb7957 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-10T22:40:08,214 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411102d3c0f4d1ae94508a31dca5da3a70663_74244978b603a65b5f2074c624cb7957 is 50, key is test_row_0/A:col10/1731278408204/Put/seqid=0 2024-11-10T22:40:08,216 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:08,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34536 deadline: 1731278468212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:08,216 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:08,217 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:08,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34566 deadline: 1731278468212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:08,217 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:08,217 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:08,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742374_1550 (size=14594) 2024-11-10T22:40:08,217 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:08,217 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 849d0d71310f,43753,1731278250663 for 'test_row_2' in 74244978b603a65b5f2074c624cb7957 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2164 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-10T22:40:08,218 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:08,218 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:08,218 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:08,218 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:08,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34584 deadline: 1731278468214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:08,218 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:08,218 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:08,219 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:08,219 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:08,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34576 deadline: 1731278468214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:08,219 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:08,219 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:08,219 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:08,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34580 deadline: 1731278468214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:08,219 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:08,219 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:08,219 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:08,219 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:08,221 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411102d3c0f4d1ae94508a31dca5da3a70663_74244978b603a65b5f2074c624cb7957 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411102d3c0f4d1ae94508a31dca5da3a70663_74244978b603a65b5f2074c624cb7957 2024-11-10T22:40:08,222 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/A/96936ada754c434c9626573765010d21, store: [table=TestAcidGuarantees family=A region=74244978b603a65b5f2074c624cb7957] 2024-11-10T22:40:08,223 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/A/96936ada754c434c9626573765010d21 is 175, key is test_row_0/A:col10/1731278408204/Put/seqid=0 2024-11-10T22:40:08,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742375_1551 (size=39549) 2024-11-10T22:40:08,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-10T22:40:08,288 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-10T22:40:08,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. 2024-11-10T22:40:08,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. as already flushing 2024-11-10T22:40:08,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. 2024-11-10T22:40:08,289 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 74244978b603a65b5f2074c624cb7957, NAME => 'TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:40:08,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 74244978b603a65b5f2074c624cb7957, NAME => 'TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:40:08,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 74244978b603a65b5f2074c624cb7957, NAME => 'TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 74244978b603a65b5f2074c624cb7957, NAME => 'TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:40:08,324 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:08,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34536 deadline: 1731278468322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:08,325 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:08,325 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:08,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34576 deadline: 1731278468322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:08,325 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:08,325 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:08,325 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:08,325 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:08,325 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:08,325 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:08,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34584 deadline: 1731278468322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:08,325 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:08,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34580 deadline: 1731278468322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:08,325 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:08,325 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:08,326 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:08,326 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:08,326 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:08,326 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:08,442 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-10T22:40:08,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. 2024-11-10T22:40:08,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. as already flushing 2024-11-10T22:40:08,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. 2024-11-10T22:40:08,442 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 74244978b603a65b5f2074c624cb7957, NAME => 'TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:40:08,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 74244978b603a65b5f2074c624cb7957, NAME => 'TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:40:08,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 74244978b603a65b5f2074c624cb7957, NAME => 'TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 74244978b603a65b5f2074c624cb7957, NAME => 'TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:40:08,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-10T22:40:08,533 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:08,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34536 deadline: 1731278468532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:08,534 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:08,534 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:08,534 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:08,534 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:08,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34576 deadline: 1731278468532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:08,534 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:08,534 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:08,534 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:08,534 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:08,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34584 deadline: 1731278468532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:08,535 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:08,535 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:08,535 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:08,535 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:08,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34580 deadline: 1731278468532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:08,535 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:08,535 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:08,535 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:08,595 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-10T22:40:08,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. 2024-11-10T22:40:08,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. as already flushing 2024-11-10T22:40:08,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. 2024-11-10T22:40:08,595 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 74244978b603a65b5f2074c624cb7957, NAME => 'TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:40:08,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 74244978b603a65b5f2074c624cb7957, NAME => 'TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:40:08,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 74244978b603a65b5f2074c624cb7957, NAME => 'TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 74244978b603a65b5f2074c624cb7957, NAME => 'TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:40:08,627 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=121, memsize=55.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/A/96936ada754c434c9626573765010d21 2024-11-10T22:40:08,634 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/B/860d981fe0e74056be6e75fb76e51402 is 50, key is test_row_0/B:col10/1731278408204/Put/seqid=0 2024-11-10T22:40:08,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742376_1552 (size=12001) 2024-11-10T22:40:08,638 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/B/860d981fe0e74056be6e75fb76e51402 2024-11-10T22:40:08,645 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/C/097462359a8943d88ed4fbd1ce86e5cb is 50, key is test_row_0/C:col10/1731278408204/Put/seqid=0 2024-11-10T22:40:08,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742377_1553 (size=12001) 2024-11-10T22:40:08,747 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-10T22:40:08,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. 2024-11-10T22:40:08,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. as already flushing 2024-11-10T22:40:08,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. 2024-11-10T22:40:08,748 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 74244978b603a65b5f2074c624cb7957, NAME => 'TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:40:08,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 74244978b603a65b5f2074c624cb7957, NAME => 'TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:40:08,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 74244978b603a65b5f2074c624cb7957, NAME => 'TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 74244978b603a65b5f2074c624cb7957, NAME => 'TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:40:08,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-10T22:40:08,845 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:08,845 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:08,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34536 deadline: 1731278468842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:08,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34576 deadline: 1731278468842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:08,846 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:08,846 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:08,846 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:08,846 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:08,846 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:08,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34580 deadline: 1731278468842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:08,846 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:08,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34584 deadline: 1731278468842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:08,846 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:08,846 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:08,846 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:08,846 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:08,846 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:08,846 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:08,846 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:08,846 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:08,901 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-10T22:40:08,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. 2024-11-10T22:40:08,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. as already flushing 2024-11-10T22:40:08,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. 2024-11-10T22:40:08,901 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 74244978b603a65b5f2074c624cb7957, NAME => 'TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:40:08,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 74244978b603a65b5f2074c624cb7957, NAME => 'TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:40:08,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 74244978b603a65b5f2074c624cb7957, NAME => 'TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 74244978b603a65b5f2074c624cb7957, NAME => 'TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:40:09,049 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/C/097462359a8943d88ed4fbd1ce86e5cb 2024-11-10T22:40:09,053 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/A/96936ada754c434c9626573765010d21 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/A/96936ada754c434c9626573765010d21 2024-11-10T22:40:09,054 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-10T22:40:09,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. 2024-11-10T22:40:09,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. as already flushing 2024-11-10T22:40:09,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. 2024-11-10T22:40:09,054 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 74244978b603a65b5f2074c624cb7957, NAME => 'TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:40:09,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 74244978b603a65b5f2074c624cb7957, NAME => 'TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:40:09,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 74244978b603a65b5f2074c624cb7957, NAME => 'TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 74244978b603a65b5f2074c624cb7957, NAME => 'TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:40:09,056 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/A/96936ada754c434c9626573765010d21, entries=200, sequenceid=121, filesize=38.6 K 2024-11-10T22:40:09,057 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/B/860d981fe0e74056be6e75fb76e51402 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/B/860d981fe0e74056be6e75fb76e51402 2024-11-10T22:40:09,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,060 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/B/860d981fe0e74056be6e75fb76e51402, entries=150, sequenceid=121, filesize=11.7 K 2024-11-10T22:40:09,060 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/C/097462359a8943d88ed4fbd1ce86e5cb as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/C/097462359a8943d88ed4fbd1ce86e5cb 2024-11-10T22:40:09,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,064 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/C/097462359a8943d88ed4fbd1ce86e5cb, entries=150, sequenceid=121, filesize=11.7 K 2024-11-10T22:40:09,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,064 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=33.54 KB/34350 for 74244978b603a65b5f2074c624cb7957 in 858ms, sequenceid=121, compaction requested=true 2024-11-10T22:40:09,065 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 74244978b603a65b5f2074c624cb7957: 2024-11-10T22:40:09,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,065 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 74244978b603a65b5f2074c624cb7957:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T22:40:09,065 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T22:40:09,065 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 74244978b603a65b5f2074c624cb7957:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T22:40:09,065 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T22:40:09,065 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T22:40:09,065 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 74244978b603a65b5f2074c624cb7957:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T22:40:09,065 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T22:40:09,065 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T22:40:09,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,066 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101597 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T22:40:09,066 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HStore(1541): 74244978b603a65b5f2074c624cb7957/A is initiating minor compaction (all files) 2024-11-10T22:40:09,066 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T22:40:09,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,066 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1541): 74244978b603a65b5f2074c624cb7957/B is initiating minor compaction (all files) 2024-11-10T22:40:09,066 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 74244978b603a65b5f2074c624cb7957/A in TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. 2024-11-10T22:40:09,066 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 74244978b603a65b5f2074c624cb7957/B in TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. 2024-11-10T22:40:09,066 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/A/313baba61c5849e99603982923b0e38b, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/A/9289d78bd70d42a6901cb57c7d742a95, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/A/96936ada754c434c9626573765010d21] into tmpdir=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp, totalSize=99.2 K 2024-11-10T22:40:09,066 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/B/0f14e05a94c149d897979563f931d484, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/B/0204fa2d7c374c04a71d60ea8e01011c, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/B/860d981fe0e74056be6e75fb76e51402] into tmpdir=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp, totalSize=35.3 K 2024-11-10T22:40:09,066 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. 2024-11-10T22:40:09,066 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. files: [hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/A/313baba61c5849e99603982923b0e38b, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/A/9289d78bd70d42a6901cb57c7d742a95, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/A/96936ada754c434c9626573765010d21] 2024-11-10T22:40:09,066 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting 0f14e05a94c149d897979563f931d484, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1731278403525 2024-11-10T22:40:09,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,066 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 313baba61c5849e99603982923b0e38b, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1731278403525 2024-11-10T22:40:09,066 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting 0204fa2d7c374c04a71d60ea8e01011c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1731278403863 2024-11-10T22:40:09,067 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9289d78bd70d42a6901cb57c7d742a95, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1731278403863 2024-11-10T22:40:09,067 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting 860d981fe0e74056be6e75fb76e51402, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1731278408204 2024-11-10T22:40:09,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,067 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 96936ada754c434c9626573765010d21, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1731278406052 2024-11-10T22:40:09,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,073 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=74244978b603a65b5f2074c624cb7957] 2024-11-10T22:40:09,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,073 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 74244978b603a65b5f2074c624cb7957#B#compaction#464 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T22:40:09,073 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/B/e45c5d066a5440b28dd7e7b6396047f3 is 50, key is test_row_0/B:col10/1731278408204/Put/seqid=0 2024-11-10T22:40:09,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,074 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241110fd097266625f4956815a57466daee4dd_74244978b603a65b5f2074c624cb7957 store=[table=TestAcidGuarantees family=A region=74244978b603a65b5f2074c624cb7957] 2024-11-10T22:40:09,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,076 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241110fd097266625f4956815a57466daee4dd_74244978b603a65b5f2074c624cb7957, store=[table=TestAcidGuarantees family=A region=74244978b603a65b5f2074c624cb7957] 2024-11-10T22:40:09,076 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110fd097266625f4956815a57466daee4dd_74244978b603a65b5f2074c624cb7957 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=74244978b603a65b5f2074c624cb7957] 2024-11-10T22:40:09,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742378_1554 (size=12241) 2024-11-10T22:40:09,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742379_1555 (size=4469) 2024-11-10T22:40:09,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,207 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-10T22:40:09,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. 2024-11-10T22:40:09,207 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2902): Flushing 74244978b603a65b5f2074c624cb7957 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-10T22:40:09,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110f14d130e4cc64f7c9c217f3b894ffca2_74244978b603a65b5f2074c624cb7957 is 50, key is test_row_0/A:col10/1731278408212/Put/seqid=0 2024-11-10T22:40:09,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742380_1556 (size=9714) 2024-11-10T22:40:09,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,222 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110f14d130e4cc64f7c9c217f3b894ffca2_74244978b603a65b5f2074c624cb7957 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110f14d130e4cc64f7c9c217f3b894ffca2_74244978b603a65b5f2074c624cb7957 2024-11-10T22:40:09,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/A/8567490a380e4889b067365d303a47e3, store: [table=TestAcidGuarantees family=A region=74244978b603a65b5f2074c624cb7957] 2024-11-10T22:40:09,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/A/8567490a380e4889b067365d303a47e3 is 175, key is test_row_0/A:col10/1731278408212/Put/seqid=0 2024-11-10T22:40:09,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742381_1557 (size=22361) 2024-11-10T22:40:09,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,231 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=129, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/A/8567490a380e4889b067365d303a47e3 2024-11-10T22:40:09,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/B/801b04ffce4146908318d927de243dd3 is 50, key is test_row_0/B:col10/1731278408212/Put/seqid=0 2024-11-10T22:40:09,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742382_1558 (size=9657) 2024-11-10T22:40:09,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,258 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/B/801b04ffce4146908318d927de243dd3 2024-11-10T22:40:09,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/C/f0fc695ba3324d359a84fbb9d6e46530 is 50, key is test_row_0/C:col10/1731278408212/Put/seqid=0 2024-11-10T22:40:09,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742383_1559 (size=9657) 2024-11-10T22:40:09,269 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/C/f0fc695ba3324d359a84fbb9d6e46530 2024-11-10T22:40:09,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-10T22:40:09,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/A/8567490a380e4889b067365d303a47e3 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/A/8567490a380e4889b067365d303a47e3 2024-11-10T22:40:09,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,279 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/A/8567490a380e4889b067365d303a47e3, entries=100, sequenceid=129, filesize=21.8 K 2024-11-10T22:40:09,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/B/801b04ffce4146908318d927de243dd3 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/B/801b04ffce4146908318d927de243dd3 2024-11-10T22:40:09,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,283 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/B/801b04ffce4146908318d927de243dd3, entries=100, sequenceid=129, filesize=9.4 K 2024-11-10T22:40:09,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/C/f0fc695ba3324d359a84fbb9d6e46530 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/C/f0fc695ba3324d359a84fbb9d6e46530 2024-11-10T22:40:09,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,287 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/C/f0fc695ba3324d359a84fbb9d6e46530, entries=100, sequenceid=129, filesize=9.4 K 2024-11-10T22:40:09,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,288 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 74244978b603a65b5f2074c624cb7957 in 81ms, sequenceid=129, compaction requested=true 2024-11-10T22:40:09,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2603): Flush status journal for 74244978b603a65b5f2074c624cb7957: 2024-11-10T22:40:09,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. 2024-11-10T22:40:09,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=167 2024-11-10T22:40:09,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.HMaster(4169): Remote procedure done, pid=167 2024-11-10T22:40:09,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,291 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=167, resume processing ppid=166 2024-11-10T22:40:09,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,291 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=167, ppid=166, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1530 sec 2024-11-10T22:40:09,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,293 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=166, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees in 1.1580 sec 2024-11-10T22:40:09,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(8855): Flush requested on 74244978b603a65b5f2074c624cb7957 2024-11-10T22:40:09,398 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T22:40:09,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,398 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 74244978b603a65b5f2074c624cb7957 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-10T22:40:09,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,405 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110a20f1b6e64484bd386b1f81634379bd3_74244978b603a65b5f2074c624cb7957 is 50, key is test_row_0/A:col10/1731278409396/Put/seqid=0 2024-11-10T22:40:09,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742384_1560 (size=19774) 2024-11-10T22:40:09,410 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,416 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110a20f1b6e64484bd386b1f81634379bd3_74244978b603a65b5f2074c624cb7957 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110a20f1b6e64484bd386b1f81634379bd3_74244978b603a65b5f2074c624cb7957 2024-11-10T22:40:09,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,416 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/A/fbef5ed2530d4ee18c98e65cfeaeaa2d, store: [table=TestAcidGuarantees family=A region=74244978b603a65b5f2074c624cb7957] 2024-11-10T22:40:09,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,417 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/A/fbef5ed2530d4ee18c98e65cfeaeaa2d is 175, key is test_row_0/A:col10/1731278409396/Put/seqid=0 2024-11-10T22:40:09,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:09,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742385_1561 (size=57029) 2024-11-10T22:40:09,421 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=140, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/A/fbef5ed2530d4ee18c98e65cfeaeaa2d 2024-11-10T22:40:09,428 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/B/9a26618c255c41909078d0a75023b483 is 50, key is test_row_0/B:col10/1731278409396/Put/seqid=0 2024-11-10T22:40:09,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742386_1562 (size=9757) 2024-11-10T22:40:09,472 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:09,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34536 deadline: 1731278469440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:09,472 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:09,472 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:09,472 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:09,478 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:09,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34576 deadline: 1731278469472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:09,478 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:09,478 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:09,478 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:09,478 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:09,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34580 deadline: 1731278469472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:09,479 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:09,479 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:09,479 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:09,479 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:09,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34584 deadline: 1731278469472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:09,479 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:09,479 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:09,479 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:09,481 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 74244978b603a65b5f2074c624cb7957#A#compaction#465 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T22:40:09,482 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/A/e14d35e4a5044bb6a87506250ab079e8 is 175, key is test_row_0/A:col10/1731278408204/Put/seqid=0 2024-11-10T22:40:09,484 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/B/e45c5d066a5440b28dd7e7b6396047f3 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/B/e45c5d066a5440b28dd7e7b6396047f3 2024-11-10T22:40:09,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742387_1563 (size=31195) 2024-11-10T22:40:09,488 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 74244978b603a65b5f2074c624cb7957/B of 74244978b603a65b5f2074c624cb7957 into e45c5d066a5440b28dd7e7b6396047f3(size=12.0 K), total size for store is 21.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T22:40:09,489 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 74244978b603a65b5f2074c624cb7957: 2024-11-10T22:40:09,489 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., storeName=74244978b603a65b5f2074c624cb7957/B, priority=13, startTime=1731278409065; duration=0sec 2024-11-10T22:40:09,489 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T22:40:09,489 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 74244978b603a65b5f2074c624cb7957:B 2024-11-10T22:40:09,489 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T22:40:09,490 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45798 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T22:40:09,490 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1541): 74244978b603a65b5f2074c624cb7957/C is initiating minor compaction (all files) 2024-11-10T22:40:09,491 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 74244978b603a65b5f2074c624cb7957/C in TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. 2024-11-10T22:40:09,491 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/C/bacedf3bb2194116a50d978460f44dfd, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/C/ca57369bdef3452ebbb4762bca3eec0b, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/C/097462359a8943d88ed4fbd1ce86e5cb, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/C/f0fc695ba3324d359a84fbb9d6e46530] into tmpdir=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp, totalSize=44.7 K 2024-11-10T22:40:09,491 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting bacedf3bb2194116a50d978460f44dfd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1731278403525 2024-11-10T22:40:09,491 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting ca57369bdef3452ebbb4762bca3eec0b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1731278403863 2024-11-10T22:40:09,492 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting 097462359a8943d88ed4fbd1ce86e5cb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1731278408204 2024-11-10T22:40:09,492 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting f0fc695ba3324d359a84fbb9d6e46530, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1731278408210 2024-11-10T22:40:09,499 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 74244978b603a65b5f2074c624cb7957#C#compaction#471 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T22:40:09,500 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/C/0dc098c699404e9bae5409f495bd925a is 50, key is test_row_0/C:col10/1731278408212/Put/seqid=0 2024-11-10T22:40:09,508 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742388_1564 (size=12275) 2024-11-10T22:40:09,586 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:09,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34536 deadline: 1731278469583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:09,586 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:09,587 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:09,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34576 deadline: 1731278469583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:09,587 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:09,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34580 deadline: 1731278469583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:09,587 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:09,587 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:09,587 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:09,587 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:09,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34584 deadline: 1731278469583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:09,587 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:09,587 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:09,587 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:09,587 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:09,587 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:09,587 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:09,587 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:09,587 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:09,795 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:09,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34536 deadline: 1731278469793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:09,795 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:09,795 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:09,795 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:09,797 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:09,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34580 deadline: 1731278469793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:09,797 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:09,797 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:09,797 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:09,797 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:09,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34584 deadline: 1731278469793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:09,797 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:09,797 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:09,797 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:09,797 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:09,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34576 deadline: 1731278469793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:09,798 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:09,798 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:09,798 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:09,833 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=140 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/B/9a26618c255c41909078d0a75023b483 2024-11-10T22:40:09,840 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/C/d8431a76a5284f6bbf4f79a13587e982 is 50, key is test_row_0/C:col10/1731278409396/Put/seqid=0 2024-11-10T22:40:09,844 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742389_1565 (size=9757) 2024-11-10T22:40:09,892 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/A/e14d35e4a5044bb6a87506250ab079e8 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/A/e14d35e4a5044bb6a87506250ab079e8 2024-11-10T22:40:09,896 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 74244978b603a65b5f2074c624cb7957/A of 74244978b603a65b5f2074c624cb7957 into e14d35e4a5044bb6a87506250ab079e8(size=30.5 K), total size for store is 52.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T22:40:09,896 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 74244978b603a65b5f2074c624cb7957: 2024-11-10T22:40:09,896 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., storeName=74244978b603a65b5f2074c624cb7957/A, priority=13, startTime=1731278409065; duration=0sec 2024-11-10T22:40:09,896 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T22:40:09,896 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 74244978b603a65b5f2074c624cb7957:A 2024-11-10T22:40:09,912 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/C/0dc098c699404e9bae5409f495bd925a as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/C/0dc098c699404e9bae5409f495bd925a 2024-11-10T22:40:09,917 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 74244978b603a65b5f2074c624cb7957/C of 74244978b603a65b5f2074c624cb7957 into 0dc098c699404e9bae5409f495bd925a(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T22:40:09,917 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 74244978b603a65b5f2074c624cb7957: 2024-11-10T22:40:09,917 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., storeName=74244978b603a65b5f2074c624cb7957/C, priority=12, startTime=1731278409065; duration=0sec 2024-11-10T22:40:09,917 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T22:40:09,917 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 74244978b603a65b5f2074c624cb7957:C 2024-11-10T22:40:10,105 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:10,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34536 deadline: 1731278470102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:10,105 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:10,105 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:10,105 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:10,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34580 deadline: 1731278470103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:10,105 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:10,106 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:10,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34584 deadline: 1731278470103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:10,106 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:10,106 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:10,106 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:10,106 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:10,106 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:10,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34576 deadline: 1731278470103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:10,106 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:10,106 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:10,106 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:10,106 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:10,106 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:10,234 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:10,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34566 deadline: 1731278470232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:10,235 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 849d0d71310f,43753,1731278250663 for 'test_row_2' in 74244978b603a65b5f2074c624cb7957 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4181 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-10T22:40:10,235 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:10,235 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:10,235 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:10,245 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=140 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/C/d8431a76a5284f6bbf4f79a13587e982 2024-11-10T22:40:10,249 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/A/fbef5ed2530d4ee18c98e65cfeaeaa2d as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/A/fbef5ed2530d4ee18c98e65cfeaeaa2d 2024-11-10T22:40:10,256 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/A/fbef5ed2530d4ee18c98e65cfeaeaa2d, entries=300, sequenceid=140, filesize=55.7 K 2024-11-10T22:40:10,257 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/B/9a26618c255c41909078d0a75023b483 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/B/9a26618c255c41909078d0a75023b483 2024-11-10T22:40:10,260 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/B/9a26618c255c41909078d0a75023b483, entries=100, sequenceid=140, filesize=9.5 K 2024-11-10T22:40:10,261 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/C/d8431a76a5284f6bbf4f79a13587e982 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/C/d8431a76a5284f6bbf4f79a13587e982 2024-11-10T22:40:10,266 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/C/d8431a76a5284f6bbf4f79a13587e982, entries=100, sequenceid=140, filesize=9.5 K 2024-11-10T22:40:10,267 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=161.02 KB/164880 for 74244978b603a65b5f2074c624cb7957 in 869ms, sequenceid=140, compaction requested=true 2024-11-10T22:40:10,267 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 74244978b603a65b5f2074c624cb7957: 2024-11-10T22:40:10,267 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 74244978b603a65b5f2074c624cb7957:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T22:40:10,267 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T22:40:10,267 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T22:40:10,267 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 74244978b603a65b5f2074c624cb7957:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T22:40:10,267 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T22:40:10,267 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 74244978b603a65b5f2074c624cb7957:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T22:40:10,267 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T22:40:10,267 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T22:40:10,268 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T22:40:10,268 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HStore(1541): 74244978b603a65b5f2074c624cb7957/A is initiating minor compaction (all files) 2024-11-10T22:40:10,268 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 74244978b603a65b5f2074c624cb7957/A in TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. 2024-11-10T22:40:10,268 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/A/e14d35e4a5044bb6a87506250ab079e8, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/A/8567490a380e4889b067365d303a47e3, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/A/fbef5ed2530d4ee18c98e65cfeaeaa2d] into tmpdir=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp, totalSize=108.0 K 2024-11-10T22:40:10,268 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. 2024-11-10T22:40:10,268 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. files: [hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/A/e14d35e4a5044bb6a87506250ab079e8, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/A/8567490a380e4889b067365d303a47e3, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/A/fbef5ed2530d4ee18c98e65cfeaeaa2d] 2024-11-10T22:40:10,269 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 31655 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T22:40:10,269 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1541): 74244978b603a65b5f2074c624cb7957/B is initiating minor compaction (all files) 2024-11-10T22:40:10,269 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 74244978b603a65b5f2074c624cb7957/B in TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. 2024-11-10T22:40:10,269 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/B/e45c5d066a5440b28dd7e7b6396047f3, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/B/801b04ffce4146908318d927de243dd3, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/B/9a26618c255c41909078d0a75023b483] into tmpdir=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp, totalSize=30.9 K 2024-11-10T22:40:10,269 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting e14d35e4a5044bb6a87506250ab079e8, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1731278408204 2024-11-10T22:40:10,269 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting e45c5d066a5440b28dd7e7b6396047f3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1731278408204 2024-11-10T22:40:10,269 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8567490a380e4889b067365d303a47e3, keycount=100, bloomtype=ROW, size=21.8 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1731278408210 2024-11-10T22:40:10,270 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting fbef5ed2530d4ee18c98e65cfeaeaa2d, keycount=300, bloomtype=ROW, size=55.7 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1731278409371 2024-11-10T22:40:10,270 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting 801b04ffce4146908318d927de243dd3, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1731278408210 2024-11-10T22:40:10,270 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting 9a26618c255c41909078d0a75023b483, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1731278409396 2024-11-10T22:40:10,278 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 74244978b603a65b5f2074c624cb7957#B#compaction#473 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T22:40:10,279 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/B/668d43b6300e4e7d8334e5592add588b is 50, key is test_row_0/B:col10/1731278409396/Put/seqid=0 2024-11-10T22:40:10,280 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=74244978b603a65b5f2074c624cb7957] 2024-11-10T22:40:10,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-10T22:40:10,283 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T22:40:10,283 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T22:40:10,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] procedure2.ProcedureExecutor(1139): Stored pid=168, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees 2024-11-10T22:40:10,284 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111032f7ac1c54504c058e22ba6c3b928387_74244978b603a65b5f2074c624cb7957 store=[table=TestAcidGuarantees family=A region=74244978b603a65b5f2074c624cb7957] 2024-11-10T22:40:10,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-10T22:40:10,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742390_1566 (size=12443) 2024-11-10T22:40:10,286 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=168, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T22:40:10,287 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=168, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T22:40:10,287 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=169, ppid=168, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T22:40:10,288 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111032f7ac1c54504c058e22ba6c3b928387_74244978b603a65b5f2074c624cb7957, store=[table=TestAcidGuarantees family=A region=74244978b603a65b5f2074c624cb7957] 2024-11-10T22:40:10,288 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111032f7ac1c54504c058e22ba6c3b928387_74244978b603a65b5f2074c624cb7957 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=74244978b603a65b5f2074c624cb7957] 2024-11-10T22:40:10,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742391_1567 (size=4469) 2024-11-10T22:40:10,392 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-10T22:40:10,392 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-10T22:40:10,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-10T22:40:10,440 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-10T22:40:10,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. 2024-11-10T22:40:10,440 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2902): Flushing 74244978b603a65b5f2074c624cb7957 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-10T22:40:10,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110e94e578ae286451bba00186959f26208_74244978b603a65b5f2074c624cb7957 is 50, key is test_row_0/A:col10/1731278409443/Put/seqid=0 2024-11-10T22:40:10,451 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742392_1568 (size=12304) 2024-11-10T22:40:10,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-10T22:40:10,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(8855): Flush requested on 74244978b603a65b5f2074c624cb7957 2024-11-10T22:40:10,615 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. as already flushing 2024-11-10T22:40:10,622 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:10,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34576 deadline: 1731278470617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:10,623 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:10,623 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:10,623 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:10,624 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:10,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34536 deadline: 1731278470622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:10,624 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:10,624 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:10,624 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:10,628 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:10,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34580 deadline: 1731278470623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:10,628 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:10,628 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:10,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34584 deadline: 1731278470624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:10,628 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:10,628 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:10,629 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:10,629 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:10,629 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:10,690 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/B/668d43b6300e4e7d8334e5592add588b as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/B/668d43b6300e4e7d8334e5592add588b 2024-11-10T22:40:10,693 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 74244978b603a65b5f2074c624cb7957#A#compaction#474 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T22:40:10,694 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/A/d2c94d6e981f4e0582401c960b220d4e is 175, key is test_row_0/A:col10/1731278409396/Put/seqid=0 2024-11-10T22:40:10,695 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 74244978b603a65b5f2074c624cb7957/B of 74244978b603a65b5f2074c624cb7957 into 668d43b6300e4e7d8334e5592add588b(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T22:40:10,695 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 74244978b603a65b5f2074c624cb7957: 2024-11-10T22:40:10,695 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., storeName=74244978b603a65b5f2074c624cb7957/B, priority=13, startTime=1731278410267; duration=0sec 2024-11-10T22:40:10,695 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T22:40:10,695 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 74244978b603a65b5f2074c624cb7957:B 2024-11-10T22:40:10,695 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-10T22:40:10,696 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-10T22:40:10,696 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-10T22:40:10,696 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. because compaction request was cancelled 2024-11-10T22:40:10,696 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 74244978b603a65b5f2074c624cb7957:C 2024-11-10T22:40:10,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742393_1569 (size=31504) 2024-11-10T22:40:10,735 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:10,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34576 deadline: 1731278470732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:10,735 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:10,735 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:10,735 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:10,735 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:10,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34536 deadline: 1731278470732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:10,736 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:10,736 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:10,736 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:10,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34580 deadline: 1731278470732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:10,736 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:10,736 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:10,736 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:10,736 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:10,736 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:10,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34584 deadline: 1731278470732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:10,736 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:10,736 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:10,736 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:10,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:10,855 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110e94e578ae286451bba00186959f26208_74244978b603a65b5f2074c624cb7957 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110e94e578ae286451bba00186959f26208_74244978b603a65b5f2074c624cb7957 2024-11-10T22:40:10,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/A/56c8bf0c509843a48f47279f8e35d7d8, store: [table=TestAcidGuarantees family=A region=74244978b603a65b5f2074c624cb7957] 2024-11-10T22:40:10,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/A/56c8bf0c509843a48f47279f8e35d7d8 is 175, key is test_row_0/A:col10/1731278409443/Put/seqid=0 2024-11-10T22:40:10,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742394_1570 (size=31105) 2024-11-10T22:40:10,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-10T22:40:10,945 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:10,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34576 deadline: 1731278470942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:10,946 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:10,946 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:10,946 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:10,946 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:10,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34536 deadline: 1731278470942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:10,946 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:10,946 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:10,946 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:10,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34580 deadline: 1731278470942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:10,946 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:10,946 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:10,946 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:10,946 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:10,947 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:10,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34584 deadline: 1731278470942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:10,947 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:10,947 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:10,947 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:11,102 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/A/d2c94d6e981f4e0582401c960b220d4e as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/A/d2c94d6e981f4e0582401c960b220d4e 2024-11-10T22:40:11,106 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 74244978b603a65b5f2074c624cb7957/A of 74244978b603a65b5f2074c624cb7957 into d2c94d6e981f4e0582401c960b220d4e(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T22:40:11,106 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 74244978b603a65b5f2074c624cb7957: 2024-11-10T22:40:11,106 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., storeName=74244978b603a65b5f2074c624cb7957/A, priority=13, startTime=1731278410267; duration=0sec 2024-11-10T22:40:11,106 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T22:40:11,106 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 74244978b603a65b5f2074c624cb7957:A 2024-11-10T22:40:11,254 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:11,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34576 deadline: 1731278471253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:11,254 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:11,254 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:11,254 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:11,254 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:11,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34536 deadline: 1731278471253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:11,254 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:11,255 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:11,255 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:11,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34580 deadline: 1731278471253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:11,255 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:11,255 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:11,255 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:11,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34584 deadline: 1731278471253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:11,255 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:11,255 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:11,255 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:11,255 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:11,255 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:11,260 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=170, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/A/56c8bf0c509843a48f47279f8e35d7d8 2024-11-10T22:40:11,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/B/89ade9a388cd41358c4a6fa5653b3bfb is 50, key is test_row_0/B:col10/1731278409443/Put/seqid=0 2024-11-10T22:40:11,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742395_1571 (size=12151) 2024-11-10T22:40:11,280 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/B/89ade9a388cd41358c4a6fa5653b3bfb 2024-11-10T22:40:11,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/C/8e70d49d390c417a8be3be74f37e7650 is 50, key is test_row_0/C:col10/1731278409443/Put/seqid=0 2024-11-10T22:40:11,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742396_1572 (size=12151) 2024-11-10T22:40:11,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-10T22:40:11,698 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/C/8e70d49d390c417a8be3be74f37e7650 2024-11-10T22:40:11,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/A/56c8bf0c509843a48f47279f8e35d7d8 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/A/56c8bf0c509843a48f47279f8e35d7d8 2024-11-10T22:40:11,705 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/A/56c8bf0c509843a48f47279f8e35d7d8, entries=150, sequenceid=170, filesize=30.4 K 2024-11-10T22:40:11,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/B/89ade9a388cd41358c4a6fa5653b3bfb as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/B/89ade9a388cd41358c4a6fa5653b3bfb 2024-11-10T22:40:11,709 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/B/89ade9a388cd41358c4a6fa5653b3bfb, entries=150, sequenceid=170, filesize=11.9 K 2024-11-10T22:40:11,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/C/8e70d49d390c417a8be3be74f37e7650 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/C/8e70d49d390c417a8be3be74f37e7650 2024-11-10T22:40:11,714 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/C/8e70d49d390c417a8be3be74f37e7650, entries=150, sequenceid=170, filesize=11.9 K 2024-11-10T22:40:11,715 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=46.96 KB/48090 for 74244978b603a65b5f2074c624cb7957 in 1274ms, sequenceid=170, compaction requested=true 2024-11-10T22:40:11,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2603): Flush status journal for 74244978b603a65b5f2074c624cb7957: 2024-11-10T22:40:11,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. 2024-11-10T22:40:11,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=169 2024-11-10T22:40:11,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.HMaster(4169): Remote procedure done, pid=169 2024-11-10T22:40:11,718 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=169, resume processing ppid=168 2024-11-10T22:40:11,718 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=169, ppid=168, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4290 sec 2024-11-10T22:40:11,720 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=168, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees in 1.4360 sec 2024-11-10T22:40:11,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(8855): Flush requested on 74244978b603a65b5f2074c624cb7957 2024-11-10T22:40:11,764 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T22:40:11,764 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 74244978b603a65b5f2074c624cb7957 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-10T22:40:11,771 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110a82efc91d2cf446abe8dd51198e85cb1_74244978b603a65b5f2074c624cb7957 is 50, key is test_row_0/A:col10/1731278410615/Put/seqid=0 2024-11-10T22:40:11,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742397_1573 (size=12304) 2024-11-10T22:40:11,797 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:11,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34584 deadline: 1731278471792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:11,797 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:11,797 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:11,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34576 deadline: 1731278471792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:11,797 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:11,797 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:11,797 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:11,797 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:11,797 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:11,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34536 deadline: 1731278471793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:11,797 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:11,798 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:11,798 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:11,798 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:11,801 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:11,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34580 deadline: 1731278471797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:11,802 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:11,802 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:11,802 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:11,906 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:11,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34584 deadline: 1731278471902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:11,906 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:11,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34576 deadline: 1731278471903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:11,907 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:11,907 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:11,907 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:11,907 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:11,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34536 deadline: 1731278471903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:11,907 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:11,907 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:11,907 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:11,907 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:11,907 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:11,907 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:11,912 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:11,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34580 deadline: 1731278471912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:11,913 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:11,913 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:11,913 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:12,116 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:12,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34584 deadline: 1731278472112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:12,116 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:12,116 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:12,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34576 deadline: 1731278472113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:12,116 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:12,116 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:12,117 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:12,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34536 deadline: 1731278472113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:12,117 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:12,117 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:12,117 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:12,117 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:12,117 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:12,117 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:12,125 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:12,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34580 deadline: 1731278472122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:12,126 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:12,126 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:12,126 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:12,175 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:12,178 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110a82efc91d2cf446abe8dd51198e85cb1_74244978b603a65b5f2074c624cb7957 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110a82efc91d2cf446abe8dd51198e85cb1_74244978b603a65b5f2074c624cb7957 2024-11-10T22:40:12,179 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/A/5bb00252f2ac4c2ba16096aae00baf63, store: [table=TestAcidGuarantees family=A region=74244978b603a65b5f2074c624cb7957] 2024-11-10T22:40:12,179 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/A/5bb00252f2ac4c2ba16096aae00baf63 is 175, key is test_row_0/A:col10/1731278410615/Put/seqid=0 2024-11-10T22:40:12,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742398_1574 (size=31105) 2024-11-10T22:40:12,426 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:12,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34584 deadline: 1731278472422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:12,426 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:12,426 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:12,426 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:12,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34576 deadline: 1731278472422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:12,426 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:12,427 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:12,427 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:12,427 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:12,427 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:12,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34536 deadline: 1731278472422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:12,427 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:12,427 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:12,427 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:12,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-10T22:40:12,433 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T22:40:12,433 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T22:40:12,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] procedure2.ProcedureExecutor(1139): Stored pid=170, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=170, table=TestAcidGuarantees 2024-11-10T22:40:12,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-10T22:40:12,435 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=170, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=170, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T22:40:12,436 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:12,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34580 deadline: 1731278472433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:12,436 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=170, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=170, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T22:40:12,436 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=171, ppid=170, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T22:40:12,436 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:12,436 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:12,436 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:12,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-10T22:40:12,583 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=183, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/A/5bb00252f2ac4c2ba16096aae00baf63 2024-11-10T22:40:12,588 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-10T22:40:12,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. 2024-11-10T22:40:12,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. as already flushing 2024-11-10T22:40:12,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. 2024-11-10T22:40:12,588 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => 74244978b603a65b5f2074c624cb7957, NAME => 'TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:40:12,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => 74244978b603a65b5f2074c624cb7957, NAME => 'TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:40:12,589 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/B/a8d020254cd2462fa7db7fb4e5403ab0 is 50, key is test_row_0/B:col10/1731278410615/Put/seqid=0 2024-11-10T22:40:12,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 74244978b603a65b5f2074c624cb7957, NAME => 'TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 74244978b603a65b5f2074c624cb7957, NAME => 'TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:40:12,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742399_1575 (size=12151) 2024-11-10T22:40:12,741 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-10T22:40:12,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. 2024-11-10T22:40:12,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. as already flushing 2024-11-10T22:40:12,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. 2024-11-10T22:40:12,741 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => 74244978b603a65b5f2074c624cb7957, NAME => 'TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:40:12,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => 74244978b603a65b5f2074c624cb7957, NAME => 'TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:40:12,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 74244978b603a65b5f2074c624cb7957, NAME => 'TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 74244978b603a65b5f2074c624cb7957, NAME => 'TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:40:12,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-10T22:40:12,894 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-10T22:40:12,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. 2024-11-10T22:40:12,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. as already flushing 2024-11-10T22:40:12,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. 2024-11-10T22:40:12,894 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => 74244978b603a65b5f2074c624cb7957, NAME => 'TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:40:12,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => 74244978b603a65b5f2074c624cb7957, NAME => 'TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:40:12,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 74244978b603a65b5f2074c624cb7957, NAME => 'TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 74244978b603a65b5f2074c624cb7957, NAME => 'TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:40:12,935 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:12,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34584 deadline: 1731278472933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:12,935 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:12,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34576 deadline: 1731278472933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:12,935 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:12,936 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:12,936 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:12,936 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:12,936 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:12,936 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:12,944 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:12,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34536 deadline: 1731278472942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:12,944 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:12,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34580 deadline: 1731278472942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:12,944 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:12,944 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:12,944 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:12,944 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:12,944 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:12,944 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:12,994 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=183 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/B/a8d020254cd2462fa7db7fb4e5403ab0 2024-11-10T22:40:13,000 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/C/573662096e5648368dece26a3d6ee404 is 50, key is test_row_0/C:col10/1731278410615/Put/seqid=0 2024-11-10T22:40:13,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742400_1576 (size=12151) 2024-11-10T22:40:13,047 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-10T22:40:13,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. 2024-11-10T22:40:13,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. as already flushing 2024-11-10T22:40:13,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. 2024-11-10T22:40:13,047 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => 74244978b603a65b5f2074c624cb7957, NAME => 'TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:40:13,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => 74244978b603a65b5f2074c624cb7957, NAME => 'TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:40:13,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 74244978b603a65b5f2074c624cb7957, NAME => 'TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 74244978b603a65b5f2074c624cb7957, NAME => 'TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:40:13,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-10T22:40:13,200 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-10T22:40:13,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. 2024-11-10T22:40:13,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. as already flushing 2024-11-10T22:40:13,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. 2024-11-10T22:40:13,200 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => 74244978b603a65b5f2074c624cb7957, NAME => 'TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:40:13,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => 74244978b603a65b5f2074c624cb7957, NAME => 'TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:40:13,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 74244978b603a65b5f2074c624cb7957, NAME => 'TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 74244978b603a65b5f2074c624cb7957, NAME => 'TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:40:13,353 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-10T22:40:13,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. 2024-11-10T22:40:13,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. as already flushing 2024-11-10T22:40:13,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. 2024-11-10T22:40:13,353 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => 74244978b603a65b5f2074c624cb7957, NAME => 'TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:40:13,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => 74244978b603a65b5f2074c624cb7957, NAME => 'TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:40:13,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 74244978b603a65b5f2074c624cb7957, NAME => 'TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 74244978b603a65b5f2074c624cb7957, NAME => 'TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:40:13,408 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=183 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/C/573662096e5648368dece26a3d6ee404 2024-11-10T22:40:13,412 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/A/5bb00252f2ac4c2ba16096aae00baf63 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/A/5bb00252f2ac4c2ba16096aae00baf63 2024-11-10T22:40:13,415 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/A/5bb00252f2ac4c2ba16096aae00baf63, entries=150, sequenceid=183, filesize=30.4 K 2024-11-10T22:40:13,416 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/B/a8d020254cd2462fa7db7fb4e5403ab0 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/B/a8d020254cd2462fa7db7fb4e5403ab0 2024-11-10T22:40:13,419 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/B/a8d020254cd2462fa7db7fb4e5403ab0, entries=150, sequenceid=183, filesize=11.9 K 2024-11-10T22:40:13,420 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/C/573662096e5648368dece26a3d6ee404 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/C/573662096e5648368dece26a3d6ee404 2024-11-10T22:40:13,423 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/C/573662096e5648368dece26a3d6ee404, entries=150, sequenceid=183, filesize=11.9 K 2024-11-10T22:40:13,424 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 74244978b603a65b5f2074c624cb7957 in 1660ms, sequenceid=183, compaction requested=true 2024-11-10T22:40:13,424 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 74244978b603a65b5f2074c624cb7957: 2024-11-10T22:40:13,424 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 74244978b603a65b5f2074c624cb7957:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T22:40:13,424 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T22:40:13,424 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 74244978b603a65b5f2074c624cb7957:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T22:40:13,424 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T22:40:13,424 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 74244978b603a65b5f2074c624cb7957:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T22:40:13,424 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-10T22:40:13,424 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T22:40:13,425 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93714 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T22:40:13,425 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1541): 74244978b603a65b5f2074c624cb7957/A is initiating minor compaction (all files) 2024-11-10T22:40:13,425 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 74244978b603a65b5f2074c624cb7957/A in TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. 2024-11-10T22:40:13,425 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/A/d2c94d6e981f4e0582401c960b220d4e, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/A/56c8bf0c509843a48f47279f8e35d7d8, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/A/5bb00252f2ac4c2ba16096aae00baf63] into tmpdir=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp, totalSize=91.5 K 2024-11-10T22:40:13,425 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. 2024-11-10T22:40:13,425 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. files: [hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/A/d2c94d6e981f4e0582401c960b220d4e, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/A/56c8bf0c509843a48f47279f8e35d7d8, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/A/5bb00252f2ac4c2ba16096aae00baf63] 2024-11-10T22:40:13,426 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting d2c94d6e981f4e0582401c960b220d4e, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1731278408210 2024-11-10T22:40:13,426 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting 56c8bf0c509843a48f47279f8e35d7d8, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731278409436 2024-11-10T22:40:13,426 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting 5bb00252f2ac4c2ba16096aae00baf63, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1731278410615 2024-11-10T22:40:13,427 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T22:40:13,429 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46334 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T22:40:13,429 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HStore(1541): 74244978b603a65b5f2074c624cb7957/C is initiating minor compaction (all files) 2024-11-10T22:40:13,429 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 74244978b603a65b5f2074c624cb7957/C in TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. 2024-11-10T22:40:13,429 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/C/0dc098c699404e9bae5409f495bd925a, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/C/d8431a76a5284f6bbf4f79a13587e982, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/C/8e70d49d390c417a8be3be74f37e7650, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/C/573662096e5648368dece26a3d6ee404] into tmpdir=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp, totalSize=45.2 K 2024-11-10T22:40:13,429 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0dc098c699404e9bae5409f495bd925a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1731278408206 2024-11-10T22:40:13,430 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting d8431a76a5284f6bbf4f79a13587e982, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1731278409396 2024-11-10T22:40:13,430 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8e70d49d390c417a8be3be74f37e7650, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731278409436 2024-11-10T22:40:13,430 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 573662096e5648368dece26a3d6ee404, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1731278410615 2024-11-10T22:40:13,435 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=74244978b603a65b5f2074c624cb7957] 2024-11-10T22:40:13,437 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241110f594726ec47d4bb3b45d3b884a250eca_74244978b603a65b5f2074c624cb7957 store=[table=TestAcidGuarantees family=A region=74244978b603a65b5f2074c624cb7957] 2024-11-10T22:40:13,439 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241110f594726ec47d4bb3b45d3b884a250eca_74244978b603a65b5f2074c624cb7957, store=[table=TestAcidGuarantees family=A region=74244978b603a65b5f2074c624cb7957] 2024-11-10T22:40:13,439 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110f594726ec47d4bb3b45d3b884a250eca_74244978b603a65b5f2074c624cb7957 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=74244978b603a65b5f2074c624cb7957] 2024-11-10T22:40:13,446 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 74244978b603a65b5f2074c624cb7957#C#compaction#482 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T22:40:13,447 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/C/ad52d63d456d414aa7106075af23fa18 is 50, key is test_row_0/C:col10/1731278410615/Put/seqid=0 2024-11-10T22:40:13,451 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742401_1577 (size=4469) 2024-11-10T22:40:13,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742402_1578 (size=12561) 2024-11-10T22:40:13,506 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-10T22:40:13,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. 2024-11-10T22:40:13,507 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2902): Flushing 74244978b603a65b5f2074c624cb7957 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-10T22:40:13,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110e435529fa79d480ead34ba1b8e556065_74244978b603a65b5f2074c624cb7957 is 50, key is test_row_0/A:col10/1731278411791/Put/seqid=0 2024-11-10T22:40:13,517 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742403_1579 (size=12304) 2024-11-10T22:40:13,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:13,521 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110e435529fa79d480ead34ba1b8e556065_74244978b603a65b5f2074c624cb7957 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110e435529fa79d480ead34ba1b8e556065_74244978b603a65b5f2074c624cb7957 2024-11-10T22:40:13,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/A/d326aec03d63402391f443d83fcdd388, store: [table=TestAcidGuarantees family=A region=74244978b603a65b5f2074c624cb7957] 2024-11-10T22:40:13,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/A/d326aec03d63402391f443d83fcdd388 is 175, key is test_row_0/A:col10/1731278411791/Put/seqid=0 2024-11-10T22:40:13,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742404_1580 (size=31105) 2024-11-10T22:40:13,526 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=208, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/A/d326aec03d63402391f443d83fcdd388 2024-11-10T22:40:13,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/B/ce707a9fc47f4be0864d721e6fbf8a76 is 50, key is test_row_0/B:col10/1731278411791/Put/seqid=0 2024-11-10T22:40:13,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742405_1581 (size=12151) 2024-11-10T22:40:13,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-10T22:40:13,853 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 74244978b603a65b5f2074c624cb7957#A#compaction#481 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T22:40:13,854 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/A/9f700b62701b4c1c8b919e7bffa48fa9 is 175, key is test_row_0/A:col10/1731278410615/Put/seqid=0 2024-11-10T22:40:13,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742406_1582 (size=31549) 2024-11-10T22:40:13,862 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/C/ad52d63d456d414aa7106075af23fa18 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/C/ad52d63d456d414aa7106075af23fa18 2024-11-10T22:40:13,862 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/A/9f700b62701b4c1c8b919e7bffa48fa9 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/A/9f700b62701b4c1c8b919e7bffa48fa9 2024-11-10T22:40:13,871 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 74244978b603a65b5f2074c624cb7957/A of 74244978b603a65b5f2074c624cb7957 into 9f700b62701b4c1c8b919e7bffa48fa9(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T22:40:13,871 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 74244978b603a65b5f2074c624cb7957: 2024-11-10T22:40:13,871 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., storeName=74244978b603a65b5f2074c624cb7957/A, priority=13, startTime=1731278413424; duration=0sec 2024-11-10T22:40:13,871 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T22:40:13,871 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 74244978b603a65b5f2074c624cb7957:A 2024-11-10T22:40:13,871 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T22:40:13,872 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36745 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T22:40:13,872 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1541): 74244978b603a65b5f2074c624cb7957/B is initiating minor compaction (all files) 2024-11-10T22:40:13,872 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 74244978b603a65b5f2074c624cb7957/B in TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. 2024-11-10T22:40:13,872 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/B/668d43b6300e4e7d8334e5592add588b, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/B/89ade9a388cd41358c4a6fa5653b3bfb, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/B/a8d020254cd2462fa7db7fb4e5403ab0] into tmpdir=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp, totalSize=35.9 K 2024-11-10T22:40:13,872 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting 668d43b6300e4e7d8334e5592add588b, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1731278408210 2024-11-10T22:40:13,873 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 74244978b603a65b5f2074c624cb7957/C of 74244978b603a65b5f2074c624cb7957 into ad52d63d456d414aa7106075af23fa18(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T22:40:13,873 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 74244978b603a65b5f2074c624cb7957: 2024-11-10T22:40:13,873 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., storeName=74244978b603a65b5f2074c624cb7957/C, priority=12, startTime=1731278413424; duration=0sec 2024-11-10T22:40:13,873 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting 89ade9a388cd41358c4a6fa5653b3bfb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731278409436 2024-11-10T22:40:13,873 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T22:40:13,873 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 74244978b603a65b5f2074c624cb7957:C 2024-11-10T22:40:13,873 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting a8d020254cd2462fa7db7fb4e5403ab0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1731278410615 2024-11-10T22:40:13,879 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 74244978b603a65b5f2074c624cb7957#B#compaction#485 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T22:40:13,880 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/B/02b3d13f9cbf4709be56b125fe6f6819 is 50, key is test_row_0/B:col10/1731278410615/Put/seqid=0 2024-11-10T22:40:13,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742407_1583 (size=12595) 2024-11-10T22:40:13,887 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/B/02b3d13f9cbf4709be56b125fe6f6819 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/B/02b3d13f9cbf4709be56b125fe6f6819 2024-11-10T22:40:13,891 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 74244978b603a65b5f2074c624cb7957/B of 74244978b603a65b5f2074c624cb7957 into 02b3d13f9cbf4709be56b125fe6f6819(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T22:40:13,891 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 74244978b603a65b5f2074c624cb7957: 2024-11-10T22:40:13,891 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., storeName=74244978b603a65b5f2074c624cb7957/B, priority=13, startTime=1731278413424; duration=0sec 2024-11-10T22:40:13,891 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T22:40:13,891 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 74244978b603a65b5f2074c624cb7957:B 2024-11-10T22:40:13,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(8855): Flush requested on 74244978b603a65b5f2074c624cb7957 2024-11-10T22:40:13,945 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. as already flushing 2024-11-10T22:40:13,950 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/B/ce707a9fc47f4be0864d721e6fbf8a76 2024-11-10T22:40:13,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/C/cc0aa2179752406fa3ca17cc0ac9f39a is 50, key is test_row_0/C:col10/1731278411791/Put/seqid=0 2024-11-10T22:40:13,960 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:13,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34584 deadline: 1731278473956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:13,960 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:13,960 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:13,960 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:13,960 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:13,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34580 deadline: 1731278473958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:13,961 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:13,961 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:13,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742408_1584 (size=12151) 2024-11-10T22:40:13,961 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:13,961 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:13,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34576 deadline: 1731278473958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:13,961 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:13,961 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:13,961 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:13,962 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/C/cc0aa2179752406fa3ca17cc0ac9f39a 2024-11-10T22:40:13,963 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:13,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34536 deadline: 1731278473960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:13,963 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:13,963 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:13,963 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:13,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/A/d326aec03d63402391f443d83fcdd388 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/A/d326aec03d63402391f443d83fcdd388 2024-11-10T22:40:13,968 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/A/d326aec03d63402391f443d83fcdd388, entries=150, sequenceid=208, filesize=30.4 K 2024-11-10T22:40:13,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/B/ce707a9fc47f4be0864d721e6fbf8a76 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/B/ce707a9fc47f4be0864d721e6fbf8a76 2024-11-10T22:40:13,972 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/B/ce707a9fc47f4be0864d721e6fbf8a76, entries=150, sequenceid=208, filesize=11.9 K 2024-11-10T22:40:13,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/C/cc0aa2179752406fa3ca17cc0ac9f39a as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/C/cc0aa2179752406fa3ca17cc0ac9f39a 2024-11-10T22:40:13,976 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/C/cc0aa2179752406fa3ca17cc0ac9f39a, entries=150, sequenceid=208, filesize=11.9 K 2024-11-10T22:40:13,977 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 74244978b603a65b5f2074c624cb7957 in 470ms, sequenceid=208, compaction requested=false 2024-11-10T22:40:13,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2603): Flush status journal for 74244978b603a65b5f2074c624cb7957: 2024-11-10T22:40:13,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. 2024-11-10T22:40:13,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=171 2024-11-10T22:40:13,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.HMaster(4169): Remote procedure done, pid=171 2024-11-10T22:40:13,980 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=171, resume processing ppid=170 2024-11-10T22:40:13,980 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=171, ppid=170, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5430 sec 2024-11-10T22:40:13,982 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=170, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=170, table=TestAcidGuarantees in 1.5480 sec 2024-11-10T22:40:14,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(8855): Flush requested on 74244978b603a65b5f2074c624cb7957 2024-11-10T22:40:14,065 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T22:40:14,065 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 74244978b603a65b5f2074c624cb7957 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-10T22:40:14,077 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411105663d9d51f694397a672bbdcfcec5328_74244978b603a65b5f2074c624cb7957 is 50, key is test_row_0/A:col10/1731278414063/Put/seqid=0 2024-11-10T22:40:14,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742409_1585 (size=14794) 2024-11-10T22:40:14,101 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:14,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34584 deadline: 1731278474097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:14,102 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:14,102 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:14,102 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:14,102 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:14,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34580 deadline: 1731278474098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:14,103 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:14,103 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:14,103 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:14,103 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:14,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34576 deadline: 1731278474099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:14,103 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:14,103 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:14,103 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:14,103 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:14,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34536 deadline: 1731278474100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:14,104 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:14,104 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:14,104 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:14,215 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:14,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34584 deadline: 1731278474212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:14,216 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:14,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34580 deadline: 1731278474212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:14,216 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:14,216 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:14,216 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:14,216 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:14,216 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:14,216 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:14,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34576 deadline: 1731278474213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:14,216 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:14,216 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:14,216 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:14,216 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:14,216 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:14,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34536 deadline: 1731278474213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:14,216 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:14,216 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:14,216 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:14,274 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:14,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34566 deadline: 1731278474272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:14,275 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 849d0d71310f,43753,1731278250663 for 'test_row_2' in 74244978b603a65b5f2074c624cb7957 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8221 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-10T22:40:14,275 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:14,275 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:14,275 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:14,425 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:14,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34584 deadline: 1731278474423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:14,425 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:14,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34580 deadline: 1731278474423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:14,426 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:14,426 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:14,426 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:14,426 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:14,426 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:14,426 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:14,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34576 deadline: 1731278474423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:14,426 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:14,426 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:14,426 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:14,426 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:14,426 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:14,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34536 deadline: 1731278474423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:14,426 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:14,426 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:14,426 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:14,481 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:14,484 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411105663d9d51f694397a672bbdcfcec5328_74244978b603a65b5f2074c624cb7957 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411105663d9d51f694397a672bbdcfcec5328_74244978b603a65b5f2074c624cb7957 2024-11-10T22:40:14,485 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/A/88596cd3a25247289509b1f2dffd22e2, store: [table=TestAcidGuarantees family=A region=74244978b603a65b5f2074c624cb7957] 2024-11-10T22:40:14,485 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/A/88596cd3a25247289509b1f2dffd22e2 is 175, key is test_row_0/A:col10/1731278414063/Put/seqid=0 2024-11-10T22:40:14,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742410_1586 (size=39749) 2024-11-10T22:40:14,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-10T22:40:14,582 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T22:40:14,583 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T22:40:14,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] procedure2.ProcedureExecutor(1139): Stored pid=172, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=172, table=TestAcidGuarantees 2024-11-10T22:40:14,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-10T22:40:14,585 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=172, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=172, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T22:40:14,585 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=172, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=172, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T22:40:14,586 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=173, ppid=172, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T22:40:14,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-10T22:40:14,736 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:14,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34584 deadline: 1731278474732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:14,736 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:14,736 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:14,736 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:14,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34576 deadline: 1731278474732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:14,736 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:14,737 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:14,737 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:14,737 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:14,737 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:14,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34580 deadline: 1731278474733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:14,737 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:14,737 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:14,737 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:14,737 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:14,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34536 deadline: 1731278474733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:14,737 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:14,737 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:14,737 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:14,738 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=173 2024-11-10T22:40:14,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. 2024-11-10T22:40:14,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. as already flushing 2024-11-10T22:40:14,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. 2024-11-10T22:40:14,738 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] handler.RSProcedureHandler(58): pid=173 java.io.IOException: Unable to complete flush {ENCODED => 74244978b603a65b5f2074c624cb7957, NAME => 'TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:40:14,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=173 java.io.IOException: Unable to complete flush {ENCODED => 74244978b603a65b5f2074c624cb7957, NAME => 'TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:40:14,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=173 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 74244978b603a65b5f2074c624cb7957, NAME => 'TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 74244978b603a65b5f2074c624cb7957, NAME => 'TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:40:14,890 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=223, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/A/88596cd3a25247289509b1f2dffd22e2 2024-11-10T22:40:14,891 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=173 2024-11-10T22:40:14,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. 2024-11-10T22:40:14,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. as already flushing 2024-11-10T22:40:14,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. 2024-11-10T22:40:14,891 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] handler.RSProcedureHandler(58): pid=173 java.io.IOException: Unable to complete flush {ENCODED => 74244978b603a65b5f2074c624cb7957, NAME => 'TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:40:14,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=173 java.io.IOException: Unable to complete flush {ENCODED => 74244978b603a65b5f2074c624cb7957, NAME => 'TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:40:14,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=173 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 74244978b603a65b5f2074c624cb7957, NAME => 'TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 74244978b603a65b5f2074c624cb7957, NAME => 'TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:40:14,898 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/B/27c2e664e93643909b90f14623f710af is 50, key is test_row_0/B:col10/1731278414063/Put/seqid=0 2024-11-10T22:40:14,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742411_1587 (size=12151) 2024-11-10T22:40:14,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-10T22:40:15,046 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=173 2024-11-10T22:40:15,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. 2024-11-10T22:40:15,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. as already flushing 2024-11-10T22:40:15,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. 2024-11-10T22:40:15,046 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] handler.RSProcedureHandler(58): pid=173 java.io.IOException: Unable to complete flush {ENCODED => 74244978b603a65b5f2074c624cb7957, NAME => 'TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:40:15,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=173 java.io.IOException: Unable to complete flush {ENCODED => 74244978b603a65b5f2074c624cb7957, NAME => 'TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:40:15,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=173 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 74244978b603a65b5f2074c624cb7957, NAME => 'TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 74244978b603a65b5f2074c624cb7957, NAME => 'TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:40:15,199 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=173 2024-11-10T22:40:15,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. 2024-11-10T22:40:15,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. as already flushing 2024-11-10T22:40:15,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. 2024-11-10T22:40:15,199 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] handler.RSProcedureHandler(58): pid=173 java.io.IOException: Unable to complete flush {ENCODED => 74244978b603a65b5f2074c624cb7957, NAME => 'TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:40:15,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=173 java.io.IOException: Unable to complete flush {ENCODED => 74244978b603a65b5f2074c624cb7957, NAME => 'TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:40:15,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=173 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 74244978b603a65b5f2074c624cb7957, NAME => 'TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 74244978b603a65b5f2074c624cb7957, NAME => 'TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:40:15,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-10T22:40:15,246 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:15,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34584 deadline: 1731278475243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:15,246 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:15,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34576 deadline: 1731278475243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:15,246 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:15,246 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:15,247 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:15,247 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:15,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34580 deadline: 1731278475243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:15,247 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:15,247 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:15,247 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:15,247 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:15,247 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:15,247 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:15,255 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:15,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34536 deadline: 1731278475252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:15,255 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:15,255 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:15,255 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:15,303 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=223 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/B/27c2e664e93643909b90f14623f710af 2024-11-10T22:40:15,309 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/C/e789fd6415ea4a87abfc52b8ef1c781e is 50, key is test_row_0/C:col10/1731278414063/Put/seqid=0 2024-11-10T22:40:15,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742412_1588 (size=12151) 2024-11-10T22:40:15,352 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=173 2024-11-10T22:40:15,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. 2024-11-10T22:40:15,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. as already flushing 2024-11-10T22:40:15,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. 2024-11-10T22:40:15,352 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] handler.RSProcedureHandler(58): pid=173 java.io.IOException: Unable to complete flush {ENCODED => 74244978b603a65b5f2074c624cb7957, NAME => 'TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:40:15,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=173 java.io.IOException: Unable to complete flush {ENCODED => 74244978b603a65b5f2074c624cb7957, NAME => 'TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:40:15,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=173 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 74244978b603a65b5f2074c624cb7957, NAME => 'TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 74244978b603a65b5f2074c624cb7957, NAME => 'TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:40:15,504 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=173 2024-11-10T22:40:15,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. 2024-11-10T22:40:15,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. as already flushing 2024-11-10T22:40:15,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. 2024-11-10T22:40:15,505 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] handler.RSProcedureHandler(58): pid=173 java.io.IOException: Unable to complete flush {ENCODED => 74244978b603a65b5f2074c624cb7957, NAME => 'TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:40:15,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=173 java.io.IOException: Unable to complete flush {ENCODED => 74244978b603a65b5f2074c624cb7957, NAME => 'TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:40:15,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=173 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 74244978b603a65b5f2074c624cb7957, NAME => 'TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 74244978b603a65b5f2074c624cb7957, NAME => 'TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:40:15,657 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=173 2024-11-10T22:40:15,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. 2024-11-10T22:40:15,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. as already flushing 2024-11-10T22:40:15,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. 2024-11-10T22:40:15,657 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] handler.RSProcedureHandler(58): pid=173 java.io.IOException: Unable to complete flush {ENCODED => 74244978b603a65b5f2074c624cb7957, NAME => 'TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:40:15,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=173 java.io.IOException: Unable to complete flush {ENCODED => 74244978b603a65b5f2074c624cb7957, NAME => 'TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:40:15,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=173 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 74244978b603a65b5f2074c624cb7957, NAME => 'TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 74244978b603a65b5f2074c624cb7957, NAME => 'TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:40:15,713 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=223 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/C/e789fd6415ea4a87abfc52b8ef1c781e 2024-11-10T22:40:15,717 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/A/88596cd3a25247289509b1f2dffd22e2 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/A/88596cd3a25247289509b1f2dffd22e2 2024-11-10T22:40:15,720 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/A/88596cd3a25247289509b1f2dffd22e2, entries=200, sequenceid=223, filesize=38.8 K 2024-11-10T22:40:15,720 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/B/27c2e664e93643909b90f14623f710af as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/B/27c2e664e93643909b90f14623f710af 2024-11-10T22:40:15,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-10T22:40:15,724 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/B/27c2e664e93643909b90f14623f710af, entries=150, sequenceid=223, filesize=11.9 K 2024-11-10T22:40:15,724 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/C/e789fd6415ea4a87abfc52b8ef1c781e as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/C/e789fd6415ea4a87abfc52b8ef1c781e 2024-11-10T22:40:15,727 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/C/e789fd6415ea4a87abfc52b8ef1c781e, entries=150, sequenceid=223, filesize=11.9 K 2024-11-10T22:40:15,728 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 74244978b603a65b5f2074c624cb7957 in 1663ms, sequenceid=223, compaction requested=true 2024-11-10T22:40:15,728 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 74244978b603a65b5f2074c624cb7957: 2024-11-10T22:40:15,728 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 74244978b603a65b5f2074c624cb7957:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T22:40:15,728 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T22:40:15,728 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 74244978b603a65b5f2074c624cb7957:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T22:40:15,728 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T22:40:15,728 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T22:40:15,728 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 74244978b603a65b5f2074c624cb7957:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T22:40:15,728 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T22:40:15,728 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T22:40:15,729 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102403 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T22:40:15,729 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T22:40:15,729 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HStore(1541): 74244978b603a65b5f2074c624cb7957/A is initiating minor compaction (all files) 2024-11-10T22:40:15,729 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 74244978b603a65b5f2074c624cb7957/A in TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. 2024-11-10T22:40:15,729 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1541): 74244978b603a65b5f2074c624cb7957/B is initiating minor compaction (all files) 2024-11-10T22:40:15,729 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/A/9f700b62701b4c1c8b919e7bffa48fa9, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/A/d326aec03d63402391f443d83fcdd388, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/A/88596cd3a25247289509b1f2dffd22e2] into tmpdir=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp, totalSize=100.0 K 2024-11-10T22:40:15,729 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 74244978b603a65b5f2074c624cb7957/B in TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. 2024-11-10T22:40:15,729 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. 2024-11-10T22:40:15,729 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. files: [hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/A/9f700b62701b4c1c8b919e7bffa48fa9, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/A/d326aec03d63402391f443d83fcdd388, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/A/88596cd3a25247289509b1f2dffd22e2] 2024-11-10T22:40:15,729 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/B/02b3d13f9cbf4709be56b125fe6f6819, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/B/ce707a9fc47f4be0864d721e6fbf8a76, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/B/27c2e664e93643909b90f14623f710af] into tmpdir=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp, totalSize=36.0 K 2024-11-10T22:40:15,730 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9f700b62701b4c1c8b919e7bffa48fa9, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1731278410615 2024-11-10T22:40:15,730 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting 02b3d13f9cbf4709be56b125fe6f6819, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1731278410615 2024-11-10T22:40:15,730 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting ce707a9fc47f4be0864d721e6fbf8a76, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731278411791 2024-11-10T22:40:15,730 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting d326aec03d63402391f443d83fcdd388, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731278411791 2024-11-10T22:40:15,730 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 88596cd3a25247289509b1f2dffd22e2, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1731278413955 2024-11-10T22:40:15,730 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting 27c2e664e93643909b90f14623f710af, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1731278413955 2024-11-10T22:40:15,735 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=74244978b603a65b5f2074c624cb7957] 2024-11-10T22:40:15,737 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 74244978b603a65b5f2074c624cb7957#B#compaction#491 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T22:40:15,738 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/B/542b81ece6024aa3bc22151d6958c918 is 50, key is test_row_0/B:col10/1731278414063/Put/seqid=0 2024-11-10T22:40:15,738 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241110f52f0f62224141ca941bca993f02d986_74244978b603a65b5f2074c624cb7957 store=[table=TestAcidGuarantees family=A region=74244978b603a65b5f2074c624cb7957] 2024-11-10T22:40:15,740 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241110f52f0f62224141ca941bca993f02d986_74244978b603a65b5f2074c624cb7957, store=[table=TestAcidGuarantees family=A region=74244978b603a65b5f2074c624cb7957] 2024-11-10T22:40:15,740 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110f52f0f62224141ca941bca993f02d986_74244978b603a65b5f2074c624cb7957 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=74244978b603a65b5f2074c624cb7957] 2024-11-10T22:40:15,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742413_1589 (size=12697) 2024-11-10T22:40:15,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742414_1590 (size=4469) 2024-11-10T22:40:15,810 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=173 2024-11-10T22:40:15,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. 2024-11-10T22:40:15,810 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(2902): Flushing 74244978b603a65b5f2074c624cb7957 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-10T22:40:15,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110dcd91692e0a047288de3cee2f39e81ce_74244978b603a65b5f2074c624cb7957 is 50, key is test_row_0/A:col10/1731278414096/Put/seqid=0 2024-11-10T22:40:15,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742415_1591 (size=12304) 2024-11-10T22:40:15,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:15,825 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110dcd91692e0a047288de3cee2f39e81ce_74244978b603a65b5f2074c624cb7957 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110dcd91692e0a047288de3cee2f39e81ce_74244978b603a65b5f2074c624cb7957 2024-11-10T22:40:15,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/A/8f55b2bc7b6143bd8befb0e53f4f9b67, store: [table=TestAcidGuarantees family=A region=74244978b603a65b5f2074c624cb7957] 2024-11-10T22:40:15,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/A/8f55b2bc7b6143bd8befb0e53f4f9b67 is 175, key is test_row_0/A:col10/1731278414096/Put/seqid=0 2024-11-10T22:40:15,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742416_1592 (size=31105) 2024-11-10T22:40:16,148 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 74244978b603a65b5f2074c624cb7957#A#compaction#490 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T22:40:16,148 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/A/5433176809c04e3a80aab221358da7ff is 175, key is test_row_0/A:col10/1731278414063/Put/seqid=0 2024-11-10T22:40:16,150 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/B/542b81ece6024aa3bc22151d6958c918 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/B/542b81ece6024aa3bc22151d6958c918 2024-11-10T22:40:16,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742417_1593 (size=31651) 2024-11-10T22:40:16,156 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 74244978b603a65b5f2074c624cb7957/B of 74244978b603a65b5f2074c624cb7957 into 542b81ece6024aa3bc22151d6958c918(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T22:40:16,156 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 74244978b603a65b5f2074c624cb7957: 2024-11-10T22:40:16,156 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., storeName=74244978b603a65b5f2074c624cb7957/B, priority=13, startTime=1731278415728; duration=0sec 2024-11-10T22:40:16,156 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T22:40:16,156 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 74244978b603a65b5f2074c624cb7957:B 2024-11-10T22:40:16,156 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T22:40:16,157 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T22:40:16,157 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/A/5433176809c04e3a80aab221358da7ff as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/A/5433176809c04e3a80aab221358da7ff 2024-11-10T22:40:16,157 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1541): 74244978b603a65b5f2074c624cb7957/C is initiating minor compaction (all files) 2024-11-10T22:40:16,158 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 74244978b603a65b5f2074c624cb7957/C in TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. 2024-11-10T22:40:16,158 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/C/ad52d63d456d414aa7106075af23fa18, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/C/cc0aa2179752406fa3ca17cc0ac9f39a, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/C/e789fd6415ea4a87abfc52b8ef1c781e] into tmpdir=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp, totalSize=36.0 K 2024-11-10T22:40:16,158 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting ad52d63d456d414aa7106075af23fa18, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1731278410615 2024-11-10T22:40:16,159 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting cc0aa2179752406fa3ca17cc0ac9f39a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731278411791 2024-11-10T22:40:16,159 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting e789fd6415ea4a87abfc52b8ef1c781e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1731278413955 2024-11-10T22:40:16,161 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 74244978b603a65b5f2074c624cb7957/A of 74244978b603a65b5f2074c624cb7957 into 5433176809c04e3a80aab221358da7ff(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T22:40:16,161 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 74244978b603a65b5f2074c624cb7957: 2024-11-10T22:40:16,161 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., storeName=74244978b603a65b5f2074c624cb7957/A, priority=13, startTime=1731278415728; duration=0sec 2024-11-10T22:40:16,161 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T22:40:16,161 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 74244978b603a65b5f2074c624cb7957:A 2024-11-10T22:40:16,164 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 74244978b603a65b5f2074c624cb7957#C#compaction#493 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T22:40:16,165 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/C/f73616b4ec2649f6b0cafc425657b3a8 is 50, key is test_row_0/C:col10/1731278414063/Put/seqid=0 2024-11-10T22:40:16,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742418_1594 (size=12663) 2024-11-10T22:40:16,230 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=247, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/A/8f55b2bc7b6143bd8befb0e53f4f9b67 2024-11-10T22:40:16,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/B/fd5b6ac2fa084096953ce9b0a4a87e97 is 50, key is test_row_0/B:col10/1731278414096/Put/seqid=0 2024-11-10T22:40:16,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742419_1595 (size=12151) 2024-11-10T22:40:16,242 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/B/fd5b6ac2fa084096953ce9b0a4a87e97 2024-11-10T22:40:16,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/C/4e9e2017ac9947448b8e2ceddd2d90cc is 50, key is test_row_0/C:col10/1731278414096/Put/seqid=0 2024-11-10T22:40:16,251 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742420_1596 (size=12151) 2024-11-10T22:40:16,252 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/C/4e9e2017ac9947448b8e2ceddd2d90cc 2024-11-10T22:40:16,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(8855): Flush requested on 74244978b603a65b5f2074c624cb7957 2024-11-10T22:40:16,255 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. as already flushing 2024-11-10T22:40:16,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/A/8f55b2bc7b6143bd8befb0e53f4f9b67 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/A/8f55b2bc7b6143bd8befb0e53f4f9b67 2024-11-10T22:40:16,273 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/A/8f55b2bc7b6143bd8befb0e53f4f9b67, entries=150, sequenceid=247, filesize=30.4 K 2024-11-10T22:40:16,273 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:16,273 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:16,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34576 deadline: 1731278476267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:16,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34536 deadline: 1731278476267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:16,274 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:16,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/B/fd5b6ac2fa084096953ce9b0a4a87e97 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/B/fd5b6ac2fa084096953ce9b0a4a87e97 2024-11-10T22:40:16,274 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:16,274 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:16,274 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:16,274 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:16,274 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:16,277 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/B/fd5b6ac2fa084096953ce9b0a4a87e97, entries=150, sequenceid=247, filesize=11.9 K 2024-11-10T22:40:16,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/C/4e9e2017ac9947448b8e2ceddd2d90cc as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/C/4e9e2017ac9947448b8e2ceddd2d90cc 2024-11-10T22:40:16,279 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:16,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34584 deadline: 1731278476274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:16,280 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:16,280 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:16,280 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:16,280 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:16,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34580 deadline: 1731278476274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:16,280 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:16,280 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:16,280 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:16,282 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/C/4e9e2017ac9947448b8e2ceddd2d90cc, entries=150, sequenceid=247, filesize=11.9 K 2024-11-10T22:40:16,283 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=67.09 KB/68700 for 74244978b603a65b5f2074c624cb7957 in 473ms, sequenceid=247, compaction requested=false 2024-11-10T22:40:16,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(2603): Flush status journal for 74244978b603a65b5f2074c624cb7957: 2024-11-10T22:40:16,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. 2024-11-10T22:40:16,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=173 2024-11-10T22:40:16,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.HMaster(4169): Remote procedure done, pid=173 2024-11-10T22:40:16,286 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=173, resume processing ppid=172 2024-11-10T22:40:16,286 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=173, ppid=172, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6990 sec 2024-11-10T22:40:16,288 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=172, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=172, table=TestAcidGuarantees in 1.7040 sec 2024-11-10T22:40:16,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(8855): Flush requested on 74244978b603a65b5f2074c624cb7957 2024-11-10T22:40:16,385 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T22:40:16,385 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 74244978b603a65b5f2074c624cb7957 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-10T22:40:16,393 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411108f323fede0b54025b08ed3c953d3a6ba_74244978b603a65b5f2074c624cb7957 is 50, key is test_row_0/A:col10/1731278416385/Put/seqid=0 2024-11-10T22:40:16,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742421_1597 (size=12454) 2024-11-10T22:40:16,419 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:16,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34584 deadline: 1731278476414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:16,419 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:16,419 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:16,419 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:16,421 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:16,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34576 deadline: 1731278476415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:16,421 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:16,421 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:16,422 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:16,422 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:16,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34580 deadline: 1731278476416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:16,422 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:16,422 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:16,422 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:16,422 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:16,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34536 deadline: 1731278476416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:16,422 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:16,422 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:16,422 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:16,526 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:16,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34584 deadline: 1731278476523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:16,527 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:16,527 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:16,527 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:16,534 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:16,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34576 deadline: 1731278476532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:16,535 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:16,535 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:16,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34580 deadline: 1731278476532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:16,535 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:16,535 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:16,535 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:16,535 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:16,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34536 deadline: 1731278476532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:16,535 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:16,535 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:16,535 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:16,535 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:16,535 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:16,572 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/C/f73616b4ec2649f6b0cafc425657b3a8 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/C/f73616b4ec2649f6b0cafc425657b3a8 2024-11-10T22:40:16,576 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 74244978b603a65b5f2074c624cb7957/C of 74244978b603a65b5f2074c624cb7957 into f73616b4ec2649f6b0cafc425657b3a8(size=12.4 K), total size for store is 24.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T22:40:16,576 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 74244978b603a65b5f2074c624cb7957: 2024-11-10T22:40:16,576 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., storeName=74244978b603a65b5f2074c624cb7957/C, priority=13, startTime=1731278415728; duration=0sec 2024-11-10T22:40:16,576 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T22:40:16,576 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 74244978b603a65b5f2074c624cb7957:C 2024-11-10T22:40:16,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-10T22:40:16,733 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T22:40:16,734 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T22:40:16,734 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:16,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34584 deadline: 1731278476733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:16,735 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:16,735 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:16,735 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:16,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] procedure2.ProcedureExecutor(1139): Stored pid=174, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=174, table=TestAcidGuarantees 2024-11-10T22:40:16,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-11-10T22:40:16,736 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=174, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=174, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T22:40:16,736 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=174, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=174, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T22:40:16,736 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=175, ppid=174, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T22:40:16,744 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:16,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34576 deadline: 1731278476742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:16,744 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:16,744 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:16,744 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:16,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34580 deadline: 1731278476742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:16,744 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:16,745 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:16,745 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:16,745 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:16,745 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:16,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34536 deadline: 1731278476742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:16,745 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:16,745 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:16,745 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:16,798 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:16,801 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411108f323fede0b54025b08ed3c953d3a6ba_74244978b603a65b5f2074c624cb7957 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411108f323fede0b54025b08ed3c953d3a6ba_74244978b603a65b5f2074c624cb7957 2024-11-10T22:40:16,801 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/A/22c4ee09e6ac4d4ead24e42e776dcb61, store: [table=TestAcidGuarantees family=A region=74244978b603a65b5f2074c624cb7957] 2024-11-10T22:40:16,802 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/A/22c4ee09e6ac4d4ead24e42e776dcb61 is 175, key is test_row_0/A:col10/1731278416385/Put/seqid=0 2024-11-10T22:40:16,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742422_1598 (size=31255) 2024-11-10T22:40:16,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-11-10T22:40:16,888 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=175 2024-11-10T22:40:16,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. 2024-11-10T22:40:16,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. as already flushing 2024-11-10T22:40:16,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. 2024-11-10T22:40:16,889 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] handler.RSProcedureHandler(58): pid=175 java.io.IOException: Unable to complete flush {ENCODED => 74244978b603a65b5f2074c624cb7957, NAME => 'TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:40:16,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=175 java.io.IOException: Unable to complete flush {ENCODED => 74244978b603a65b5f2074c624cb7957, NAME => 'TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:40:16,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=175 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 74244978b603a65b5f2074c624cb7957, NAME => 'TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 74244978b603a65b5f2074c624cb7957, NAME => 'TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:40:17,041 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=175 2024-11-10T22:40:17,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. 2024-11-10T22:40:17,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. as already flushing 2024-11-10T22:40:17,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. 2024-11-10T22:40:17,042 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] handler.RSProcedureHandler(58): pid=175 java.io.IOException: Unable to complete flush {ENCODED => 74244978b603a65b5f2074c624cb7957, NAME => 'TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:40:17,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=175 java.io.IOException: Unable to complete flush {ENCODED => 74244978b603a65b5f2074c624cb7957, NAME => 'TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:40:17,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=175 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 74244978b603a65b5f2074c624cb7957, NAME => 'TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 74244978b603a65b5f2074c624cb7957, NAME => 'TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:40:17,045 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:17,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34584 deadline: 1731278477043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:17,045 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:17,045 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:17,045 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:17,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-11-10T22:40:17,053 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:17,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34576 deadline: 1731278477052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:17,054 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:17,053 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:17,054 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:17,054 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:17,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34580 deadline: 1731278477052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:17,054 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:17,054 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:17,054 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:17,054 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:17,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34536 deadline: 1731278477052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:17,054 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:17,054 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:17,054 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:17,194 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=175 2024-11-10T22:40:17,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. 2024-11-10T22:40:17,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. as already flushing 2024-11-10T22:40:17,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. 2024-11-10T22:40:17,195 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] handler.RSProcedureHandler(58): pid=175 java.io.IOException: Unable to complete flush {ENCODED => 74244978b603a65b5f2074c624cb7957, NAME => 'TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:40:17,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=175 java.io.IOException: Unable to complete flush {ENCODED => 74244978b603a65b5f2074c624cb7957, NAME => 'TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:40:17,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=175 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 74244978b603a65b5f2074c624cb7957, NAME => 'TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 74244978b603a65b5f2074c624cb7957, NAME => 'TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:40:17,206 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=264, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/A/22c4ee09e6ac4d4ead24e42e776dcb61 2024-11-10T22:40:17,212 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/B/9f3a64fb4b9e471ca361a37764b6f0ae is 50, key is test_row_0/B:col10/1731278416385/Put/seqid=0 2024-11-10T22:40:17,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742423_1599 (size=12301) 2024-11-10T22:40:17,347 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=175 2024-11-10T22:40:17,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. 2024-11-10T22:40:17,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. as already flushing 2024-11-10T22:40:17,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. 2024-11-10T22:40:17,347 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] handler.RSProcedureHandler(58): pid=175 java.io.IOException: Unable to complete flush {ENCODED => 74244978b603a65b5f2074c624cb7957, NAME => 'TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:40:17,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=175 java.io.IOException: Unable to complete flush {ENCODED => 74244978b603a65b5f2074c624cb7957, NAME => 'TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:40:17,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=175 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 74244978b603a65b5f2074c624cb7957, NAME => 'TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 74244978b603a65b5f2074c624cb7957, NAME => 'TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:40:17,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-11-10T22:40:17,500 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=175 2024-11-10T22:40:17,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. 2024-11-10T22:40:17,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. as already flushing 2024-11-10T22:40:17,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. 2024-11-10T22:40:17,500 ERROR [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] handler.RSProcedureHandler(58): pid=175 java.io.IOException: Unable to complete flush {ENCODED => 74244978b603a65b5f2074c624cb7957, NAME => 'TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:40:17,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=175 java.io.IOException: Unable to complete flush {ENCODED => 74244978b603a65b5f2074c624cb7957, NAME => 'TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:40:17,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.HMaster(4177): Remote procedure failed, pid=175 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 74244978b603a65b5f2074c624cb7957, NAME => 'TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 74244978b603a65b5f2074c624cb7957, NAME => 'TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T22:40:17,556 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:17,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34584 deadline: 1731278477553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:17,557 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:17,557 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:17,557 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:17,563 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:17,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34576 deadline: 1731278477562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:17,563 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:17,563 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:17,563 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:17,566 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:17,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34580 deadline: 1731278477562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:17,566 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:17,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34536 deadline: 1731278477562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:17,566 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:17,566 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:17,566 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:17,566 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:17,566 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:17,566 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:17,616 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=264 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/B/9f3a64fb4b9e471ca361a37764b6f0ae 2024-11-10T22:40:17,623 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/C/fc22149d10a1474c9135a1521d87d6df is 50, key is test_row_0/C:col10/1731278416385/Put/seqid=0 2024-11-10T22:40:17,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742424_1600 (size=12301) 2024-11-10T22:40:17,628 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=264 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/C/fc22149d10a1474c9135a1521d87d6df 2024-11-10T22:40:17,632 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/A/22c4ee09e6ac4d4ead24e42e776dcb61 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/A/22c4ee09e6ac4d4ead24e42e776dcb61 2024-11-10T22:40:17,635 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/A/22c4ee09e6ac4d4ead24e42e776dcb61, entries=150, sequenceid=264, filesize=30.5 K 2024-11-10T22:40:17,636 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/B/9f3a64fb4b9e471ca361a37764b6f0ae as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/B/9f3a64fb4b9e471ca361a37764b6f0ae 2024-11-10T22:40:17,639 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/B/9f3a64fb4b9e471ca361a37764b6f0ae, entries=150, sequenceid=264, filesize=12.0 K 2024-11-10T22:40:17,639 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/C/fc22149d10a1474c9135a1521d87d6df as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/C/fc22149d10a1474c9135a1521d87d6df 2024-11-10T22:40:17,642 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/C/fc22149d10a1474c9135a1521d87d6df, entries=150, sequenceid=264, filesize=12.0 K 2024-11-10T22:40:17,643 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=127.47 KB/130530 for 74244978b603a65b5f2074c624cb7957 in 1258ms, sequenceid=264, compaction requested=true 2024-11-10T22:40:17,643 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 74244978b603a65b5f2074c624cb7957: 2024-11-10T22:40:17,643 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 74244978b603a65b5f2074c624cb7957:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T22:40:17,643 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T22:40:17,643 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T22:40:17,643 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 74244978b603a65b5f2074c624cb7957:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T22:40:17,643 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T22:40:17,643 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 74244978b603a65b5f2074c624cb7957:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T22:40:17,643 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T22:40:17,643 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T22:40:17,644 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94011 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T22:40:17,644 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T22:40:17,644 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HStore(1541): 74244978b603a65b5f2074c624cb7957/A is initiating minor compaction (all files) 2024-11-10T22:40:17,644 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1541): 74244978b603a65b5f2074c624cb7957/B is initiating minor compaction (all files) 2024-11-10T22:40:17,644 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 74244978b603a65b5f2074c624cb7957/A in TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. 2024-11-10T22:40:17,644 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 74244978b603a65b5f2074c624cb7957/B in TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. 2024-11-10T22:40:17,644 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/B/542b81ece6024aa3bc22151d6958c918, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/B/fd5b6ac2fa084096953ce9b0a4a87e97, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/B/9f3a64fb4b9e471ca361a37764b6f0ae] into tmpdir=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp, totalSize=36.3 K 2024-11-10T22:40:17,644 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/A/5433176809c04e3a80aab221358da7ff, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/A/8f55b2bc7b6143bd8befb0e53f4f9b67, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/A/22c4ee09e6ac4d4ead24e42e776dcb61] into tmpdir=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp, totalSize=91.8 K 2024-11-10T22:40:17,644 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. 2024-11-10T22:40:17,644 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. files: [hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/A/5433176809c04e3a80aab221358da7ff, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/A/8f55b2bc7b6143bd8befb0e53f4f9b67, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/A/22c4ee09e6ac4d4ead24e42e776dcb61] 2024-11-10T22:40:17,645 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting 542b81ece6024aa3bc22151d6958c918, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1731278413955 2024-11-10T22:40:17,645 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5433176809c04e3a80aab221358da7ff, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1731278413955 2024-11-10T22:40:17,645 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting fd5b6ac2fa084096953ce9b0a4a87e97, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1731278414090 2024-11-10T22:40:17,645 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8f55b2bc7b6143bd8befb0e53f4f9b67, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1731278414090 2024-11-10T22:40:17,645 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 22c4ee09e6ac4d4ead24e42e776dcb61, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1731278416266 2024-11-10T22:40:17,645 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting 9f3a64fb4b9e471ca361a37764b6f0ae, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1731278416266 2024-11-10T22:40:17,652 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 74244978b603a65b5f2074c624cb7957#B#compaction#499 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T22:40:17,653 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/B/36e8d6d6110b4aa4b64afca826d74230 is 50, key is test_row_0/B:col10/1731278416385/Put/seqid=0 2024-11-10T22:40:17,653 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=175 2024-11-10T22:40:17,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. 2024-11-10T22:40:17,654 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegion(2902): Flushing 74244978b603a65b5f2074c624cb7957 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-10T22:40:17,655 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=74244978b603a65b5f2074c624cb7957] 2024-11-10T22:40:17,658 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241110216a902b0fa348819369a67f584ddb55_74244978b603a65b5f2074c624cb7957 store=[table=TestAcidGuarantees family=A region=74244978b603a65b5f2074c624cb7957] 2024-11-10T22:40:17,660 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241110216a902b0fa348819369a67f584ddb55_74244978b603a65b5f2074c624cb7957, store=[table=TestAcidGuarantees family=A region=74244978b603a65b5f2074c624cb7957] 2024-11-10T22:40:17,660 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110216a902b0fa348819369a67f584ddb55_74244978b603a65b5f2074c624cb7957 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=74244978b603a65b5f2074c624cb7957] 2024-11-10T22:40:17,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742425_1601 (size=12949) 2024-11-10T22:40:17,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411107ba04c79dcb046a48ddcda55f64a076e_74244978b603a65b5f2074c624cb7957 is 50, key is test_row_0/A:col10/1731278416414/Put/seqid=0 2024-11-10T22:40:17,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742426_1602 (size=4469) 2024-11-10T22:40:17,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742427_1603 (size=12454) 2024-11-10T22:40:17,687 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 74244978b603a65b5f2074c624cb7957#A#compaction#500 average throughput is 0.76 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T22:40:17,688 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/A/8bb0ba8e0e74432fa92d64f0dbf58fca is 175, key is test_row_0/A:col10/1731278416385/Put/seqid=0 2024-11-10T22:40:17,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742428_1604 (size=31903) 2024-11-10T22:40:17,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-11-10T22:40:18,070 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/B/36e8d6d6110b4aa4b64afca826d74230 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/B/36e8d6d6110b4aa4b64afca826d74230 2024-11-10T22:40:18,075 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 74244978b603a65b5f2074c624cb7957/B of 74244978b603a65b5f2074c624cb7957 into 36e8d6d6110b4aa4b64afca826d74230(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T22:40:18,075 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 74244978b603a65b5f2074c624cb7957: 2024-11-10T22:40:18,075 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., storeName=74244978b603a65b5f2074c624cb7957/B, priority=13, startTime=1731278417643; duration=0sec 2024-11-10T22:40:18,075 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T22:40:18,075 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 74244978b603a65b5f2074c624cb7957:B 2024-11-10T22:40:18,075 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T22:40:18,076 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37115 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T22:40:18,076 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1541): 74244978b603a65b5f2074c624cb7957/C is initiating minor compaction (all files) 2024-11-10T22:40:18,076 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 74244978b603a65b5f2074c624cb7957/C in TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. 2024-11-10T22:40:18,076 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/C/f73616b4ec2649f6b0cafc425657b3a8, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/C/4e9e2017ac9947448b8e2ceddd2d90cc, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/C/fc22149d10a1474c9135a1521d87d6df] into tmpdir=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp, totalSize=36.2 K 2024-11-10T22:40:18,076 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting f73616b4ec2649f6b0cafc425657b3a8, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1731278413955 2024-11-10T22:40:18,077 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting 4e9e2017ac9947448b8e2ceddd2d90cc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1731278414090 2024-11-10T22:40:18,077 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting fc22149d10a1474c9135a1521d87d6df, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1731278416266 2024-11-10T22:40:18,083 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 74244978b603a65b5f2074c624cb7957#C#compaction#502 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T22:40:18,083 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/C/ef53a3a65458426d9865d5be2aa28f20 is 50, key is test_row_0/C:col10/1731278416385/Put/seqid=0 2024-11-10T22:40:18,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742429_1605 (size=12915) 2024-11-10T22:40:18,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:18,091 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411107ba04c79dcb046a48ddcda55f64a076e_74244978b603a65b5f2074c624cb7957 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411107ba04c79dcb046a48ddcda55f64a076e_74244978b603a65b5f2074c624cb7957 2024-11-10T22:40:18,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/A/b61fd364dc6f402f95b8255cf5523d0e, store: [table=TestAcidGuarantees family=A region=74244978b603a65b5f2074c624cb7957] 2024-11-10T22:40:18,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/A/b61fd364dc6f402f95b8255cf5523d0e is 175, key is test_row_0/A:col10/1731278416414/Put/seqid=0 2024-11-10T22:40:18,095 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742430_1606 (size=31255) 2024-11-10T22:40:18,098 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/A/8bb0ba8e0e74432fa92d64f0dbf58fca as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/A/8bb0ba8e0e74432fa92d64f0dbf58fca 2024-11-10T22:40:18,103 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 74244978b603a65b5f2074c624cb7957/A of 74244978b603a65b5f2074c624cb7957 into 8bb0ba8e0e74432fa92d64f0dbf58fca(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T22:40:18,103 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 74244978b603a65b5f2074c624cb7957: 2024-11-10T22:40:18,103 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., storeName=74244978b603a65b5f2074c624cb7957/A, priority=13, startTime=1731278417643; duration=0sec 2024-11-10T22:40:18,103 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T22:40:18,103 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 74244978b603a65b5f2074c624cb7957:A 2024-11-10T22:40:18,491 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/C/ef53a3a65458426d9865d5be2aa28f20 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/C/ef53a3a65458426d9865d5be2aa28f20 2024-11-10T22:40:18,496 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=287, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/A/b61fd364dc6f402f95b8255cf5523d0e 2024-11-10T22:40:18,496 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 74244978b603a65b5f2074c624cb7957/C of 74244978b603a65b5f2074c624cb7957 into ef53a3a65458426d9865d5be2aa28f20(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T22:40:18,496 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 74244978b603a65b5f2074c624cb7957: 2024-11-10T22:40:18,496 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., storeName=74244978b603a65b5f2074c624cb7957/C, priority=13, startTime=1731278417643; duration=0sec 2024-11-10T22:40:18,496 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T22:40:18,496 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 74244978b603a65b5f2074c624cb7957:C 2024-11-10T22:40:18,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/B/cc842154b05f48339fea58268b2072f3 is 50, key is test_row_0/B:col10/1731278416414/Put/seqid=0 2024-11-10T22:40:18,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742431_1607 (size=12301) 2024-11-10T22:40:18,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(8855): Flush requested on 74244978b603a65b5f2074c624cb7957 2024-11-10T22:40:18,576 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. as already flushing 2024-11-10T22:40:18,591 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:18,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34576 deadline: 1731278478585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:18,592 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:18,592 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:18,592 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:18,593 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:18,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34584 deadline: 1731278478589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:18,593 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:18,593 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:18,593 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:18,595 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:18,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34536 deadline: 1731278478590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:18,595 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:18,596 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:18,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34580 deadline: 1731278478591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:18,596 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:18,596 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:18,596 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:18,596 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:18,596 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:18,705 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:18,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34576 deadline: 1731278478702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:18,705 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:18,705 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:18,706 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:18,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34584 deadline: 1731278478702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:18,706 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:18,706 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:18,706 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:18,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34536 deadline: 1731278478702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:18,706 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:18,706 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:18,706 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:18,706 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:18,706 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:18,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34580 deadline: 1731278478702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:18,706 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:18,706 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:18,706 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:18,706 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:18,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-11-10T22:40:18,905 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=287 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/B/cc842154b05f48339fea58268b2072f3 2024-11-10T22:40:18,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/C/4ed331405a8f489f86167e6c107ab8b7 is 50, key is test_row_0/C:col10/1731278416414/Put/seqid=0 2024-11-10T22:40:18,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742432_1608 (size=12301) 2024-11-10T22:40:18,915 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:18,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34576 deadline: 1731278478912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:18,915 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:18,915 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:18,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34584 deadline: 1731278478912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:18,915 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:18,915 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:18,916 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:18,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34536 deadline: 1731278478912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:18,916 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:18,916 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:18,916 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:18,916 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:18,916 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:18,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34580 deadline: 1731278478912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:18,916 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:18,916 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:18,916 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:18,916 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:18,916 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:19,225 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:19,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34576 deadline: 1731278479223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:19,226 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:19,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34536 deadline: 1731278479223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:19,226 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:19,226 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:19,226 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:19,226 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:19,226 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:19,226 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:19,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34584 deadline: 1731278479223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:19,226 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:19,226 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:19,226 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:19,226 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:19,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34580 deadline: 1731278479223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:19,226 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:19,227 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:19,227 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:19,227 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:19,315 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=287 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/C/4ed331405a8f489f86167e6c107ab8b7 2024-11-10T22:40:19,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/A/b61fd364dc6f402f95b8255cf5523d0e as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/A/b61fd364dc6f402f95b8255cf5523d0e 2024-11-10T22:40:19,323 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/A/b61fd364dc6f402f95b8255cf5523d0e, entries=150, sequenceid=287, filesize=30.5 K 2024-11-10T22:40:19,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/B/cc842154b05f48339fea58268b2072f3 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/B/cc842154b05f48339fea58268b2072f3 2024-11-10T22:40:19,327 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/B/cc842154b05f48339fea58268b2072f3, entries=150, sequenceid=287, filesize=12.0 K 2024-11-10T22:40:19,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/C/4ed331405a8f489f86167e6c107ab8b7 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/C/4ed331405a8f489f86167e6c107ab8b7 2024-11-10T22:40:19,330 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/C/4ed331405a8f489f86167e6c107ab8b7, entries=150, sequenceid=287, filesize=12.0 K 2024-11-10T22:40:19,331 INFO [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 74244978b603a65b5f2074c624cb7957 in 1678ms, sequenceid=287, compaction requested=false 2024-11-10T22:40:19,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegion(2603): Flush status journal for 74244978b603a65b5f2074c624cb7957: 2024-11-10T22:40:19,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. 2024-11-10T22:40:19,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849d0d71310f:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=175 2024-11-10T22:40:19,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.HMaster(4169): Remote procedure done, pid=175 2024-11-10T22:40:19,334 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=175, resume processing ppid=174 2024-11-10T22:40:19,334 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=175, ppid=174, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5970 sec 2024-11-10T22:40:19,336 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=174, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=174, table=TestAcidGuarantees in 2.6010 sec 2024-11-10T22:40:19,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(8855): Flush requested on 74244978b603a65b5f2074c624cb7957 2024-11-10T22:40:19,735 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T22:40:19,735 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 74244978b603a65b5f2074c624cb7957 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-10T22:40:19,745 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111005212343fb6441e0b2fd60ee273486cb_74244978b603a65b5f2074c624cb7957 is 50, key is test_row_0/A:col10/1731278419735/Put/seqid=0 2024-11-10T22:40:19,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742433_1609 (size=12454) 2024-11-10T22:40:19,759 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:19,762 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111005212343fb6441e0b2fd60ee273486cb_74244978b603a65b5f2074c624cb7957 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111005212343fb6441e0b2fd60ee273486cb_74244978b603a65b5f2074c624cb7957 2024-11-10T22:40:19,763 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/A/5fcb329703ad42c0824d3138bf46c17b, store: [table=TestAcidGuarantees family=A region=74244978b603a65b5f2074c624cb7957] 2024-11-10T22:40:19,763 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/A/5fcb329703ad42c0824d3138bf46c17b is 175, key is test_row_0/A:col10/1731278419735/Put/seqid=0 2024-11-10T22:40:19,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742434_1610 (size=31255) 2024-11-10T22:40:19,768 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=307, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/A/5fcb329703ad42c0824d3138bf46c17b 2024-11-10T22:40:19,768 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:19,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34580 deadline: 1731278479761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:19,768 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:19,768 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:19,768 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:19,768 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:19,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34536 deadline: 1731278479762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:19,769 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:19,769 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:19,769 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:19,774 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/B/b5d610445f4244b5a989cf8aa882b614 is 50, key is test_row_0/B:col10/1731278419735/Put/seqid=0 2024-11-10T22:40:19,775 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:19,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34584 deadline: 1731278479768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:19,775 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:19,775 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:19,775 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:19,775 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:19,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34576 deadline: 1731278479768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:19,776 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:19,776 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:19,776 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:19,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742435_1611 (size=12301) 2024-11-10T22:40:19,875 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:19,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34580 deadline: 1731278479872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:19,875 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:19,876 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:19,876 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:19,876 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:19,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34536 deadline: 1731278479872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:19,876 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:19,876 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:19,876 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:19,886 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:19,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34584 deadline: 1731278479883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:19,887 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:19,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34576 deadline: 1731278479883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:19,887 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:19,887 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:19,887 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:19,887 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:19,887 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:19,887 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:20,083 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:20,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34580 deadline: 1731278480082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:20,083 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:20,083 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:20,083 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:20,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34536 deadline: 1731278480082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:20,083 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:20,084 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:20,084 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:20,084 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:20,097 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:20,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34584 deadline: 1731278480093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:20,097 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:20,097 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:20,097 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:20,097 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:20,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34576 deadline: 1731278480093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:20,097 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:20,097 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:20,097 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:20,178 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=307 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/B/b5d610445f4244b5a989cf8aa882b614 2024-11-10T22:40:20,186 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/C/3b1343f578d24eacbfc6af72614bfd75 is 50, key is test_row_0/C:col10/1731278419735/Put/seqid=0 2024-11-10T22:40:20,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742436_1612 (size=12301) 2024-11-10T22:40:20,195 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=307 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/C/3b1343f578d24eacbfc6af72614bfd75 2024-11-10T22:40:20,198 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/A/5fcb329703ad42c0824d3138bf46c17b as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/A/5fcb329703ad42c0824d3138bf46c17b 2024-11-10T22:40:20,202 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/A/5fcb329703ad42c0824d3138bf46c17b, entries=150, sequenceid=307, filesize=30.5 K 2024-11-10T22:40:20,202 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/B/b5d610445f4244b5a989cf8aa882b614 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/B/b5d610445f4244b5a989cf8aa882b614 2024-11-10T22:40:20,205 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/B/b5d610445f4244b5a989cf8aa882b614, entries=150, sequenceid=307, filesize=12.0 K 2024-11-10T22:40:20,206 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/C/3b1343f578d24eacbfc6af72614bfd75 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/C/3b1343f578d24eacbfc6af72614bfd75 2024-11-10T22:40:20,209 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/C/3b1343f578d24eacbfc6af72614bfd75, entries=150, sequenceid=307, filesize=12.0 K 2024-11-10T22:40:20,210 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=114.05 KB/116790 for 74244978b603a65b5f2074c624cb7957 in 475ms, sequenceid=307, compaction requested=true 2024-11-10T22:40:20,210 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 74244978b603a65b5f2074c624cb7957: 2024-11-10T22:40:20,210 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 74244978b603a65b5f2074c624cb7957:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T22:40:20,210 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T22:40:20,210 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T22:40:20,210 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 74244978b603a65b5f2074c624cb7957:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T22:40:20,210 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T22:40:20,210 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T22:40:20,210 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 74244978b603a65b5f2074c624cb7957:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T22:40:20,210 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T22:40:20,211 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T22:40:20,211 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94413 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T22:40:20,211 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1541): 74244978b603a65b5f2074c624cb7957/B is initiating minor compaction (all files) 2024-11-10T22:40:20,211 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HStore(1541): 74244978b603a65b5f2074c624cb7957/A is initiating minor compaction (all files) 2024-11-10T22:40:20,211 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 74244978b603a65b5f2074c624cb7957/B in TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. 2024-11-10T22:40:20,211 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 74244978b603a65b5f2074c624cb7957/A in TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. 2024-11-10T22:40:20,211 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/B/36e8d6d6110b4aa4b64afca826d74230, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/B/cc842154b05f48339fea58268b2072f3, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/B/b5d610445f4244b5a989cf8aa882b614] into tmpdir=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp, totalSize=36.7 K 2024-11-10T22:40:20,211 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/A/8bb0ba8e0e74432fa92d64f0dbf58fca, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/A/b61fd364dc6f402f95b8255cf5523d0e, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/A/5fcb329703ad42c0824d3138bf46c17b] into tmpdir=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp, totalSize=92.2 K 2024-11-10T22:40:20,211 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. 2024-11-10T22:40:20,211 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. files: [hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/A/8bb0ba8e0e74432fa92d64f0dbf58fca, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/A/b61fd364dc6f402f95b8255cf5523d0e, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/A/5fcb329703ad42c0824d3138bf46c17b] 2024-11-10T22:40:20,211 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting 36e8d6d6110b4aa4b64afca826d74230, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1731278416266 2024-11-10T22:40:20,212 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting cc842154b05f48339fea58268b2072f3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1731278416413 2024-11-10T22:40:20,212 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8bb0ba8e0e74432fa92d64f0dbf58fca, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1731278416266 2024-11-10T22:40:20,212 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting b5d610445f4244b5a989cf8aa882b614, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=307, earliestPutTs=1731278418588 2024-11-10T22:40:20,212 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting b61fd364dc6f402f95b8255cf5523d0e, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1731278416413 2024-11-10T22:40:20,213 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5fcb329703ad42c0824d3138bf46c17b, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=307, earliestPutTs=1731278418588 2024-11-10T22:40:20,219 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 74244978b603a65b5f2074c624cb7957#B#compaction#508 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T22:40:20,219 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/B/1d9e8d83a44d407a82e27c62f3222b16 is 50, key is test_row_0/B:col10/1731278419735/Put/seqid=0 2024-11-10T22:40:20,220 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=74244978b603a65b5f2074c624cb7957] 2024-11-10T22:40:20,222 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111097e59927ebb74a7894b291a3e143af06_74244978b603a65b5f2074c624cb7957 store=[table=TestAcidGuarantees family=A region=74244978b603a65b5f2074c624cb7957] 2024-11-10T22:40:20,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742437_1613 (size=13051) 2024-11-10T22:40:20,223 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111097e59927ebb74a7894b291a3e143af06_74244978b603a65b5f2074c624cb7957, store=[table=TestAcidGuarantees family=A region=74244978b603a65b5f2074c624cb7957] 2024-11-10T22:40:20,223 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111097e59927ebb74a7894b291a3e143af06_74244978b603a65b5f2074c624cb7957 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=74244978b603a65b5f2074c624cb7957] 2024-11-10T22:40:20,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742438_1614 (size=4469) 2024-11-10T22:40:20,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(8855): Flush requested on 74244978b603a65b5f2074c624cb7957 2024-11-10T22:40:20,395 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T22:40:20,395 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 74244978b603a65b5f2074c624cb7957 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-10T22:40:20,402 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411100e8d2180d8f5473899114e5e80a51652_74244978b603a65b5f2074c624cb7957 is 50, key is test_row_0/A:col10/1731278419766/Put/seqid=0 2024-11-10T22:40:20,405 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742439_1615 (size=14994) 2024-11-10T22:40:20,406 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:20,409 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411100e8d2180d8f5473899114e5e80a51652_74244978b603a65b5f2074c624cb7957 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411100e8d2180d8f5473899114e5e80a51652_74244978b603a65b5f2074c624cb7957 2024-11-10T22:40:20,410 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/A/d0b2da00267449a0a239c334df436bd8, store: [table=TestAcidGuarantees family=A region=74244978b603a65b5f2074c624cb7957] 2024-11-10T22:40:20,410 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/A/d0b2da00267449a0a239c334df436bd8 is 175, key is test_row_0/A:col10/1731278419766/Put/seqid=0 2024-11-10T22:40:20,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742440_1616 (size=39949) 2024-11-10T22:40:20,425 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=329, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/A/d0b2da00267449a0a239c334df436bd8 2024-11-10T22:40:20,431 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/B/ed1fa467480743909b8b7672389c25f3 is 50, key is test_row_0/B:col10/1731278419766/Put/seqid=0 2024-11-10T22:40:20,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742441_1617 (size=12301) 2024-11-10T22:40:20,451 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:20,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34584 deadline: 1731278480417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:20,451 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:20,451 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:20,451 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:20,456 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:20,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34536 deadline: 1731278480451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:20,457 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:20,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34576 deadline: 1731278480451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:20,457 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:20,457 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:20,457 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:20,457 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:20,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34580 deadline: 1731278480451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:20,457 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:20,457 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:20,457 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:20,457 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:20,457 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:20,457 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:20,565 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:20,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34584 deadline: 1731278480562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:20,566 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:20,566 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:20,566 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:20,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34536 deadline: 1731278480562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:20,566 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:20,566 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:20,566 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:20,566 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:20,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34576 deadline: 1731278480562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:20,566 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:20,566 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:20,566 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:20,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34580 deadline: 1731278480562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:20,567 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:20,567 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:20,567 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:20,567 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:20,567 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:20,627 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/B/1d9e8d83a44d407a82e27c62f3222b16 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/B/1d9e8d83a44d407a82e27c62f3222b16 2024-11-10T22:40:20,627 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 74244978b603a65b5f2074c624cb7957#A#compaction#509 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T22:40:20,628 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/A/33c945fa5df44e238d7955533050f24f is 175, key is test_row_0/A:col10/1731278419735/Put/seqid=0 2024-11-10T22:40:20,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742442_1618 (size=32005) 2024-11-10T22:40:20,632 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 74244978b603a65b5f2074c624cb7957/B of 74244978b603a65b5f2074c624cb7957 into 1d9e8d83a44d407a82e27c62f3222b16(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T22:40:20,632 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 74244978b603a65b5f2074c624cb7957: 2024-11-10T22:40:20,632 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., storeName=74244978b603a65b5f2074c624cb7957/B, priority=13, startTime=1731278420210; duration=0sec 2024-11-10T22:40:20,632 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T22:40:20,632 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 74244978b603a65b5f2074c624cb7957:B 2024-11-10T22:40:20,632 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T22:40:20,633 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T22:40:20,633 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1541): 74244978b603a65b5f2074c624cb7957/C is initiating minor compaction (all files) 2024-11-10T22:40:20,633 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 74244978b603a65b5f2074c624cb7957/C in TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. 2024-11-10T22:40:20,633 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/C/ef53a3a65458426d9865d5be2aa28f20, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/C/4ed331405a8f489f86167e6c107ab8b7, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/C/3b1343f578d24eacbfc6af72614bfd75] into tmpdir=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp, totalSize=36.6 K 2024-11-10T22:40:20,633 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting ef53a3a65458426d9865d5be2aa28f20, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1731278416266 2024-11-10T22:40:20,634 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting 4ed331405a8f489f86167e6c107ab8b7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1731278416413 2024-11-10T22:40:20,634 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting 3b1343f578d24eacbfc6af72614bfd75, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=307, earliestPutTs=1731278418588 2024-11-10T22:40:20,640 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 74244978b603a65b5f2074c624cb7957#C#compaction#512 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T22:40:20,640 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/C/05f485b15db94b55a2d475d50bacd1ff is 50, key is test_row_0/C:col10/1731278419735/Put/seqid=0 2024-11-10T22:40:20,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742443_1619 (size=13017) 2024-11-10T22:40:20,679 INFO [Thread-2357 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2357. 2024-11-10T22:40:20,679 DEBUG [Thread-2357 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T22:40:20,679 DEBUG [Thread-2357 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:40:20,679 DEBUG [Thread-2357 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:40:20,679 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T22:40:20,679 INFO [Thread-2359 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2359. 2024-11-10T22:40:20,679 DEBUG [Thread-2359 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T22:40:20,679 DEBUG [Thread-2359 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:40:20,679 DEBUG [Thread-2359 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:40:20,680 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T22:40:20,681 INFO [Thread-2365 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2365. 2024-11-10T22:40:20,681 DEBUG [Thread-2365 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T22:40:20,681 DEBUG [Thread-2365 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:40:20,681 DEBUG [Thread-2365 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:40:20,682 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T22:40:20,683 INFO [Thread-2363 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2363. 2024-11-10T22:40:20,683 DEBUG [Thread-2363 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T22:40:20,683 DEBUG [Thread-2363 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:40:20,683 DEBUG [Thread-2363 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:40:20,683 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T22:40:20,684 INFO [Thread-2361 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2361. 2024-11-10T22:40:20,684 DEBUG [Thread-2361 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T22:40:20,684 DEBUG [Thread-2361 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:40:20,684 DEBUG [Thread-2361 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:40:20,684 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T22:40:20,772 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:20,772 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:20,772 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:20,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34584 deadline: 1731278480772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:20,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34576 deadline: 1731278480772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:20,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34536 deadline: 1731278480772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:20,772 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:20,773 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:20,773 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:20,773 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:20,773 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:20,773 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:20,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34580 deadline: 1731278480772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:20,773 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:20,773 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:20,773 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:20,773 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:20,773 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:20,773 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:20,773 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:20,836 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=329 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/B/ed1fa467480743909b8b7672389c25f3 2024-11-10T22:40:20,841 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/C/44fad965582845b28d4e779eaf31f10a is 50, key is test_row_0/C:col10/1731278419766/Put/seqid=0 2024-11-10T22:40:20,844 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742444_1620 (size=12301) 2024-11-10T22:40:20,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-11-10T22:40:20,902 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T22:40:21,035 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/A/33c945fa5df44e238d7955533050f24f as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/A/33c945fa5df44e238d7955533050f24f 2024-11-10T22:40:21,038 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 74244978b603a65b5f2074c624cb7957/A of 74244978b603a65b5f2074c624cb7957 into 33c945fa5df44e238d7955533050f24f(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T22:40:21,038 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 74244978b603a65b5f2074c624cb7957: 2024-11-10T22:40:21,038 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., storeName=74244978b603a65b5f2074c624cb7957/A, priority=13, startTime=1731278420210; duration=0sec 2024-11-10T22:40:21,038 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T22:40:21,038 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 74244978b603a65b5f2074c624cb7957:A 2024-11-10T22:40:21,047 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/C/05f485b15db94b55a2d475d50bacd1ff as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/C/05f485b15db94b55a2d475d50bacd1ff 2024-11-10T22:40:21,050 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 74244978b603a65b5f2074c624cb7957/C of 74244978b603a65b5f2074c624cb7957 into 05f485b15db94b55a2d475d50bacd1ff(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T22:40:21,050 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 74244978b603a65b5f2074c624cb7957: 2024-11-10T22:40:21,050 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., storeName=74244978b603a65b5f2074c624cb7957/C, priority=13, startTime=1731278420210; duration=0sec 2024-11-10T22:40:21,050 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T22:40:21,050 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 74244978b603a65b5f2074c624cb7957:C 2024-11-10T22:40:21,083 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:21,083 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:21,083 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:21,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34584 deadline: 1731278481082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:21,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34536 deadline: 1731278481083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:21,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34576 deadline: 1731278481083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:21,083 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T22:40:21,083 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:21,083 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:21,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43753 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34580 deadline: 1731278481083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 2024-11-10T22:40:21,083 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:21,083 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:21,083 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:21,083 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:21,083 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:21,083 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:21,083 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:21,083 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 , the old value is region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:21,083 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T22:40:21,083 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., hostname=849d0d71310f,43753,1731278250663, seqNum=5 because the exception is null or not the one we care about 2024-11-10T22:40:21,245 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=329 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/C/44fad965582845b28d4e779eaf31f10a 2024-11-10T22:40:21,248 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/A/d0b2da00267449a0a239c334df436bd8 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/A/d0b2da00267449a0a239c334df436bd8 2024-11-10T22:40:21,251 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/A/d0b2da00267449a0a239c334df436bd8, entries=200, sequenceid=329, filesize=39.0 K 2024-11-10T22:40:21,251 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/B/ed1fa467480743909b8b7672389c25f3 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/B/ed1fa467480743909b8b7672389c25f3 2024-11-10T22:40:21,253 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/B/ed1fa467480743909b8b7672389c25f3, entries=150, sequenceid=329, filesize=12.0 K 2024-11-10T22:40:21,254 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/C/44fad965582845b28d4e779eaf31f10a as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/C/44fad965582845b28d4e779eaf31f10a 2024-11-10T22:40:21,256 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/C/44fad965582845b28d4e779eaf31f10a, entries=150, sequenceid=329, filesize=12.0 K 2024-11-10T22:40:21,257 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=87.22 KB/89310 for 74244978b603a65b5f2074c624cb7957 in 862ms, sequenceid=329, compaction requested=false 2024-11-10T22:40:21,257 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 74244978b603a65b5f2074c624cb7957: 2024-11-10T22:40:21,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43753 {}] regionserver.HRegion(8855): Flush requested on 74244978b603a65b5f2074c624cb7957 2024-11-10T22:40:21,594 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T22:40:21,594 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 74244978b603a65b5f2074c624cb7957 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-10T22:40:21,594 INFO [Thread-2348 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2348. 2024-11-10T22:40:21,594 DEBUG [Thread-2348 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T22:40:21,594 DEBUG [Thread-2348 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:40:21,594 INFO [Thread-2350 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2350. 2024-11-10T22:40:21,594 INFO [Thread-2352 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2352. 2024-11-10T22:40:21,594 DEBUG [Thread-2350 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T22:40:21,594 DEBUG [Thread-2352 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T22:40:21,594 DEBUG [Thread-2350 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:40:21,594 DEBUG [Thread-2352 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:40:21,594 DEBUG [Thread-2348 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:40:21,594 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T22:40:21,594 DEBUG [Thread-2352 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:40:21,594 DEBUG [Thread-2350 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:40:21,595 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T22:40:21,595 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T22:40:21,596 INFO [Thread-2354 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2354. 2024-11-10T22:40:21,596 DEBUG [Thread-2354 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T22:40:21,596 DEBUG [Thread-2354 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:40:21,596 DEBUG [Thread-2354 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:40:21,596 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T22:40:21,600 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411102693968e03bf44b0aff72f0537a69825_74244978b603a65b5f2074c624cb7957 is 50, key is test_row_0/A:col10/1731278421593/Put/seqid=0 2024-11-10T22:40:21,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742445_1621 (size=12454) 2024-11-10T22:40:22,003 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:22,006 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411102693968e03bf44b0aff72f0537a69825_74244978b603a65b5f2074c624cb7957 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411102693968e03bf44b0aff72f0537a69825_74244978b603a65b5f2074c624cb7957 2024-11-10T22:40:22,006 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/A/fa5df9fffc994140b6c7023cc37d1331, store: [table=TestAcidGuarantees family=A region=74244978b603a65b5f2074c624cb7957] 2024-11-10T22:40:22,007 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/A/fa5df9fffc994140b6c7023cc37d1331 is 175, key is test_row_0/A:col10/1731278421593/Put/seqid=0 2024-11-10T22:40:22,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742446_1622 (size=31255) 2024-11-10T22:40:22,410 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=351, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/A/fa5df9fffc994140b6c7023cc37d1331 2024-11-10T22:40:22,415 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/B/cea867e6e9184561940cb89e3fd30930 is 50, key is test_row_0/B:col10/1731278421593/Put/seqid=0 2024-11-10T22:40:22,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742447_1623 (size=12301) 2024-11-10T22:40:22,819 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=351 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/B/cea867e6e9184561940cb89e3fd30930 2024-11-10T22:40:22,824 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/C/af3d41ce544f4674bccf87c2d844fb2b is 50, key is test_row_0/C:col10/1731278421593/Put/seqid=0 2024-11-10T22:40:22,827 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742448_1624 (size=12301) 2024-11-10T22:40:23,228 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=351 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/C/af3d41ce544f4674bccf87c2d844fb2b 2024-11-10T22:40:23,231 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/A/fa5df9fffc994140b6c7023cc37d1331 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/A/fa5df9fffc994140b6c7023cc37d1331 2024-11-10T22:40:23,234 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/A/fa5df9fffc994140b6c7023cc37d1331, entries=150, sequenceid=351, filesize=30.5 K 2024-11-10T22:40:23,234 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/B/cea867e6e9184561940cb89e3fd30930 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/B/cea867e6e9184561940cb89e3fd30930 2024-11-10T22:40:23,236 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/B/cea867e6e9184561940cb89e3fd30930, entries=150, sequenceid=351, filesize=12.0 K 2024-11-10T22:40:23,237 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/C/af3d41ce544f4674bccf87c2d844fb2b as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/C/af3d41ce544f4674bccf87c2d844fb2b 2024-11-10T22:40:23,239 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/C/af3d41ce544f4674bccf87c2d844fb2b, entries=150, sequenceid=351, filesize=12.0 K 2024-11-10T22:40:23,240 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=6.71 KB/6870 for 74244978b603a65b5f2074c624cb7957 in 1646ms, sequenceid=351, compaction requested=true 2024-11-10T22:40:23,240 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 74244978b603a65b5f2074c624cb7957: 2024-11-10T22:40:23,240 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 74244978b603a65b5f2074c624cb7957:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T22:40:23,240 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T22:40:23,240 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 74244978b603a65b5f2074c624cb7957:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T22:40:23,240 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T22:40:23,240 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T22:40:23,240 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 74244978b603a65b5f2074c624cb7957:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T22:40:23,240 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T22:40:23,240 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T22:40:23,241 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T22:40:23,241 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T22:40:23,241 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HStore(1541): 74244978b603a65b5f2074c624cb7957/A is initiating minor compaction (all files) 2024-11-10T22:40:23,241 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1541): 74244978b603a65b5f2074c624cb7957/B is initiating minor compaction (all files) 2024-11-10T22:40:23,241 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 74244978b603a65b5f2074c624cb7957/A in TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. 2024-11-10T22:40:23,241 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 74244978b603a65b5f2074c624cb7957/B in TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. 2024-11-10T22:40:23,241 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/A/33c945fa5df44e238d7955533050f24f, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/A/d0b2da00267449a0a239c334df436bd8, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/A/fa5df9fffc994140b6c7023cc37d1331] into tmpdir=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp, totalSize=100.8 K 2024-11-10T22:40:23,241 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/B/1d9e8d83a44d407a82e27c62f3222b16, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/B/ed1fa467480743909b8b7672389c25f3, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/B/cea867e6e9184561940cb89e3fd30930] into tmpdir=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp, totalSize=36.8 K 2024-11-10T22:40:23,241 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. 2024-11-10T22:40:23,241 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. files: [hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/A/33c945fa5df44e238d7955533050f24f, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/A/d0b2da00267449a0a239c334df436bd8, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/A/fa5df9fffc994140b6c7023cc37d1331] 2024-11-10T22:40:23,241 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting 1d9e8d83a44d407a82e27c62f3222b16, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=307, earliestPutTs=1731278418588 2024-11-10T22:40:23,241 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 33c945fa5df44e238d7955533050f24f, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=307, earliestPutTs=1731278418588 2024-11-10T22:40:23,241 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting ed1fa467480743909b8b7672389c25f3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1731278419752 2024-11-10T22:40:23,241 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting d0b2da00267449a0a239c334df436bd8, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1731278419752 2024-11-10T22:40:23,241 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting cea867e6e9184561940cb89e3fd30930, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1731278420407 2024-11-10T22:40:23,241 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] compactions.Compactor(225): Compacting fa5df9fffc994140b6c7023cc37d1331, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1731278420407 2024-11-10T22:40:23,247 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=74244978b603a65b5f2074c624cb7957] 2024-11-10T22:40:23,247 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 74244978b603a65b5f2074c624cb7957#B#compaction#517 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T22:40:23,247 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/B/654c8ae0be1f41e198b766e0060ce254 is 50, key is test_row_0/B:col10/1731278421593/Put/seqid=0 2024-11-10T22:40:23,248 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111076af07c17cb34b76a9268c0d5fa3cf11_74244978b603a65b5f2074c624cb7957 store=[table=TestAcidGuarantees family=A region=74244978b603a65b5f2074c624cb7957] 2024-11-10T22:40:23,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742449_1625 (size=13153) 2024-11-10T22:40:23,250 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111076af07c17cb34b76a9268c0d5fa3cf11_74244978b603a65b5f2074c624cb7957, store=[table=TestAcidGuarantees family=A region=74244978b603a65b5f2074c624cb7957] 2024-11-10T22:40:23,251 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111076af07c17cb34b76a9268c0d5fa3cf11_74244978b603a65b5f2074c624cb7957 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=74244978b603a65b5f2074c624cb7957] 2024-11-10T22:40:23,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742450_1626 (size=4469) 2024-11-10T22:40:23,254 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 74244978b603a65b5f2074c624cb7957#A#compaction#518 average throughput is 3.49 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T22:40:23,254 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/A/cb1574506b394cf7915477e70a330c35 is 175, key is test_row_0/A:col10/1731278421593/Put/seqid=0 2024-11-10T22:40:23,256 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742451_1627 (size=32107) 2024-11-10T22:40:23,655 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/B/654c8ae0be1f41e198b766e0060ce254 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/B/654c8ae0be1f41e198b766e0060ce254 2024-11-10T22:40:23,659 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 74244978b603a65b5f2074c624cb7957/B of 74244978b603a65b5f2074c624cb7957 into 654c8ae0be1f41e198b766e0060ce254(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T22:40:23,659 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 74244978b603a65b5f2074c624cb7957: 2024-11-10T22:40:23,659 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., storeName=74244978b603a65b5f2074c624cb7957/B, priority=13, startTime=1731278423240; duration=0sec 2024-11-10T22:40:23,659 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T22:40:23,659 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 74244978b603a65b5f2074c624cb7957:B 2024-11-10T22:40:23,659 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T22:40:23,660 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/A/cb1574506b394cf7915477e70a330c35 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/A/cb1574506b394cf7915477e70a330c35 2024-11-10T22:40:23,660 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T22:40:23,660 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1541): 74244978b603a65b5f2074c624cb7957/C is initiating minor compaction (all files) 2024-11-10T22:40:23,660 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 74244978b603a65b5f2074c624cb7957/C in TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. 2024-11-10T22:40:23,661 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/C/05f485b15db94b55a2d475d50bacd1ff, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/C/44fad965582845b28d4e779eaf31f10a, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/C/af3d41ce544f4674bccf87c2d844fb2b] into tmpdir=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp, totalSize=36.7 K 2024-11-10T22:40:23,661 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting 05f485b15db94b55a2d475d50bacd1ff, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=307, earliestPutTs=1731278418588 2024-11-10T22:40:23,661 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting 44fad965582845b28d4e779eaf31f10a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1731278419752 2024-11-10T22:40:23,662 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] compactions.Compactor(225): Compacting af3d41ce544f4674bccf87c2d844fb2b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1731278420407 2024-11-10T22:40:23,664 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 74244978b603a65b5f2074c624cb7957/A of 74244978b603a65b5f2074c624cb7957 into cb1574506b394cf7915477e70a330c35(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T22:40:23,664 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 74244978b603a65b5f2074c624cb7957: 2024-11-10T22:40:23,664 INFO [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., storeName=74244978b603a65b5f2074c624cb7957/A, priority=13, startTime=1731278423240; duration=0sec 2024-11-10T22:40:23,664 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T22:40:23,664 DEBUG [RS:0;849d0d71310f:43753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 74244978b603a65b5f2074c624cb7957:A 2024-11-10T22:40:23,668 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 74244978b603a65b5f2074c624cb7957#C#compaction#519 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T22:40:23,668 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/C/1712c96782754560af0803805e05ff11 is 50, key is test_row_0/C:col10/1731278421593/Put/seqid=0 2024-11-10T22:40:23,671 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742452_1628 (size=13119) 2024-11-10T22:40:24,075 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/C/1712c96782754560af0803805e05ff11 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/C/1712c96782754560af0803805e05ff11 2024-11-10T22:40:24,080 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 74244978b603a65b5f2074c624cb7957/C of 74244978b603a65b5f2074c624cb7957 into 1712c96782754560af0803805e05ff11(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T22:40:24,080 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 74244978b603a65b5f2074c624cb7957: 2024-11-10T22:40:24,080 INFO [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., storeName=74244978b603a65b5f2074c624cb7957/C, priority=13, startTime=1731278423240; duration=0sec 2024-11-10T22:40:24,080 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T22:40:24,080 DEBUG [RS:0;849d0d71310f:43753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 74244978b603a65b5f2074c624cb7957:C 2024-11-10T22:40:24,334 INFO [Thread-2346 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2346. 2024-11-10T22:40:24,335 DEBUG [Thread-2346 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T22:40:24,335 DEBUG [Thread-2346 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:40:24,335 DEBUG [Thread-2346 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:40:24,335 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T22:40:24,335 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-10T22:40:24,335 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 20 2024-11-10T22:40:24,335 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 62 2024-11-10T22:40:24,335 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 68 2024-11-10T22:40:24,336 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 61 2024-11-10T22:40:24,336 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 64 2024-11-10T22:40:24,336 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-10T22:40:24,336 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3254 2024-11-10T22:40:24,336 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3264 2024-11-10T22:40:24,336 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3375 2024-11-10T22:40:24,336 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3245 2024-11-10T22:40:24,336 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3273 2024-11-10T22:40:24,336 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-10T22:40:24,336 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-10T22:40:24,336 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T22:40:24,336 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:40:24,336 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:40:24,336 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T22:40:24,337 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-10T22:40:24,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] procedure2.ProcedureExecutor(1139): Stored pid=176, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-10T22:40:24,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=176 2024-11-10T22:40:24,341 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731278424341"}]},"ts":"1731278424341"} 2024-11-10T22:40:24,343 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-10T22:40:24,343 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-10T22:40:24,344 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=177, ppid=176, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-10T22:40:24,346 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=178, ppid=177, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=74244978b603a65b5f2074c624cb7957, UNASSIGN}] 2024-11-10T22:40:24,346 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=178, ppid=177, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=74244978b603a65b5f2074c624cb7957, UNASSIGN 2024-11-10T22:40:24,347 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=178 updating hbase:meta row=74244978b603a65b5f2074c624cb7957, regionState=CLOSING, regionLocation=849d0d71310f,43753,1731278250663 2024-11-10T22:40:24,349 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=178, ppid=177, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=74244978b603a65b5f2074c624cb7957, UNASSIGN because future has completed 2024-11-10T22:40:24,349 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-10T22:40:24,349 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=179, ppid=178, state=RUNNABLE, hasLock=false; CloseRegionProcedure 74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663}] 2024-11-10T22:40:24,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=176 2024-11-10T22:40:24,502 INFO [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] handler.UnassignRegionHandler(122): Close 74244978b603a65b5f2074c624cb7957 2024-11-10T22:40:24,502 DEBUG [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-10T22:40:24,502 DEBUG [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] regionserver.HRegion(1722): Closing 74244978b603a65b5f2074c624cb7957, disabling compactions & flushes 2024-11-10T22:40:24,502 INFO [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. 2024-11-10T22:40:24,502 DEBUG [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. 2024-11-10T22:40:24,502 DEBUG [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. after waiting 0 ms 2024-11-10T22:40:24,502 DEBUG [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. 2024-11-10T22:40:24,502 INFO [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] regionserver.HRegion(2902): Flushing 74244978b603a65b5f2074c624cb7957 3/3 column families, dataSize=13.42 KB heapSize=35.91 KB 2024-11-10T22:40:24,507 DEBUG [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411109bd027f30c03421e8e19a33d3f4e4245_74244978b603a65b5f2074c624cb7957 is 50, key is test_row_2/A:col10/1731278424333/Put/seqid=0 2024-11-10T22:40:24,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742453_1629 (size=7374) 2024-11-10T22:40:24,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=176 2024-11-10T22:40:24,911 DEBUG [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T22:40:24,914 INFO [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411109bd027f30c03421e8e19a33d3f4e4245_74244978b603a65b5f2074c624cb7957 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411109bd027f30c03421e8e19a33d3f4e4245_74244978b603a65b5f2074c624cb7957 2024-11-10T22:40:24,915 DEBUG [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/A/0a8aa1ea6ab64ae68dc0eb843c2d9ed0, store: [table=TestAcidGuarantees family=A region=74244978b603a65b5f2074c624cb7957] 2024-11-10T22:40:24,915 DEBUG [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/A/0a8aa1ea6ab64ae68dc0eb843c2d9ed0 is 175, key is test_row_2/A:col10/1731278424333/Put/seqid=0 2024-11-10T22:40:24,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742454_1630 (size=13865) 2024-11-10T22:40:24,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=176 2024-11-10T22:40:25,319 INFO [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=359, memsize=4.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/A/0a8aa1ea6ab64ae68dc0eb843c2d9ed0 2024-11-10T22:40:25,324 DEBUG [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/B/a2befa0e8a4b48bf86422a3c1a63333d is 50, key is test_row_2/B:col10/1731278424333/Put/seqid=0 2024-11-10T22:40:25,331 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742455_1631 (size=7415) 2024-11-10T22:40:25,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=176 2024-11-10T22:40:25,731 INFO [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=359 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/B/a2befa0e8a4b48bf86422a3c1a63333d 2024-11-10T22:40:25,738 DEBUG [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/C/1cde88902b1242c7bc104bd3eb41b3a7 is 50, key is test_row_2/C:col10/1731278424333/Put/seqid=0 2024-11-10T22:40:25,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742456_1632 (size=7415) 2024-11-10T22:40:26,142 INFO [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=359 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/C/1cde88902b1242c7bc104bd3eb41b3a7 2024-11-10T22:40:26,145 DEBUG [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/A/0a8aa1ea6ab64ae68dc0eb843c2d9ed0 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/A/0a8aa1ea6ab64ae68dc0eb843c2d9ed0 2024-11-10T22:40:26,147 INFO [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/A/0a8aa1ea6ab64ae68dc0eb843c2d9ed0, entries=50, sequenceid=359, filesize=13.5 K 2024-11-10T22:40:26,148 DEBUG [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/B/a2befa0e8a4b48bf86422a3c1a63333d as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/B/a2befa0e8a4b48bf86422a3c1a63333d 2024-11-10T22:40:26,150 INFO [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/B/a2befa0e8a4b48bf86422a3c1a63333d, entries=50, sequenceid=359, filesize=7.2 K 2024-11-10T22:40:26,151 DEBUG [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/.tmp/C/1cde88902b1242c7bc104bd3eb41b3a7 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/C/1cde88902b1242c7bc104bd3eb41b3a7 2024-11-10T22:40:26,154 INFO [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/C/1cde88902b1242c7bc104bd3eb41b3a7, entries=50, sequenceid=359, filesize=7.2 K 2024-11-10T22:40:26,155 INFO [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] regionserver.HRegion(3140): Finished flush of dataSize ~13.42 KB/13740, heapSize ~35.86 KB/36720, currentSize=0 B/0 for 74244978b603a65b5f2074c624cb7957 in 1653ms, sequenceid=359, compaction requested=false 2024-11-10T22:40:26,156 DEBUG [StoreCloser-TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/A/bbe3f0bfc4024a6187077b135f1a8e0c, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/A/aeff6581916c45eca3665e7cc3ed057f, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/A/e68d4ca0bd184424851426c45d6574a5, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/A/313baba61c5849e99603982923b0e38b, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/A/9b9bec2a61aa494ea8350881b2ed94a4, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/A/9289d78bd70d42a6901cb57c7d742a95, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/A/96936ada754c434c9626573765010d21, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/A/e14d35e4a5044bb6a87506250ab079e8, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/A/8567490a380e4889b067365d303a47e3, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/A/fbef5ed2530d4ee18c98e65cfeaeaa2d, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/A/d2c94d6e981f4e0582401c960b220d4e, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/A/56c8bf0c509843a48f47279f8e35d7d8, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/A/9f700b62701b4c1c8b919e7bffa48fa9, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/A/5bb00252f2ac4c2ba16096aae00baf63, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/A/d326aec03d63402391f443d83fcdd388, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/A/88596cd3a25247289509b1f2dffd22e2, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/A/5433176809c04e3a80aab221358da7ff, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/A/8f55b2bc7b6143bd8befb0e53f4f9b67, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/A/8bb0ba8e0e74432fa92d64f0dbf58fca, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/A/22c4ee09e6ac4d4ead24e42e776dcb61, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/A/b61fd364dc6f402f95b8255cf5523d0e, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/A/33c945fa5df44e238d7955533050f24f, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/A/5fcb329703ad42c0824d3138bf46c17b, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/A/d0b2da00267449a0a239c334df436bd8, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/A/fa5df9fffc994140b6c7023cc37d1331] to archive 2024-11-10T22:40:26,156 DEBUG [StoreCloser-TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T22:40:26,158 DEBUG [StoreCloser-TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/A/bbe3f0bfc4024a6187077b135f1a8e0c to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/A/bbe3f0bfc4024a6187077b135f1a8e0c 2024-11-10T22:40:26,158 DEBUG [StoreCloser-TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/A/aeff6581916c45eca3665e7cc3ed057f to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/A/aeff6581916c45eca3665e7cc3ed057f 2024-11-10T22:40:26,159 DEBUG [StoreCloser-TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/A/e68d4ca0bd184424851426c45d6574a5 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/A/e68d4ca0bd184424851426c45d6574a5 2024-11-10T22:40:26,160 DEBUG [StoreCloser-TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/A/313baba61c5849e99603982923b0e38b to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/A/313baba61c5849e99603982923b0e38b 2024-11-10T22:40:26,161 DEBUG [StoreCloser-TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/A/9b9bec2a61aa494ea8350881b2ed94a4 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/A/9b9bec2a61aa494ea8350881b2ed94a4 2024-11-10T22:40:26,162 DEBUG [StoreCloser-TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/A/9289d78bd70d42a6901cb57c7d742a95 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/A/9289d78bd70d42a6901cb57c7d742a95 2024-11-10T22:40:26,163 DEBUG [StoreCloser-TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/A/96936ada754c434c9626573765010d21 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/A/96936ada754c434c9626573765010d21 2024-11-10T22:40:26,163 DEBUG [StoreCloser-TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/A/e14d35e4a5044bb6a87506250ab079e8 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/A/e14d35e4a5044bb6a87506250ab079e8 2024-11-10T22:40:26,164 DEBUG [StoreCloser-TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/A/8567490a380e4889b067365d303a47e3 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/A/8567490a380e4889b067365d303a47e3 2024-11-10T22:40:26,165 DEBUG [StoreCloser-TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/A/fbef5ed2530d4ee18c98e65cfeaeaa2d to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/A/fbef5ed2530d4ee18c98e65cfeaeaa2d 2024-11-10T22:40:26,165 DEBUG [StoreCloser-TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/A/d2c94d6e981f4e0582401c960b220d4e to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/A/d2c94d6e981f4e0582401c960b220d4e 2024-11-10T22:40:26,166 DEBUG [StoreCloser-TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/A/56c8bf0c509843a48f47279f8e35d7d8 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/A/56c8bf0c509843a48f47279f8e35d7d8 2024-11-10T22:40:26,167 DEBUG [StoreCloser-TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/A/9f700b62701b4c1c8b919e7bffa48fa9 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/A/9f700b62701b4c1c8b919e7bffa48fa9 2024-11-10T22:40:26,168 DEBUG [StoreCloser-TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/A/5bb00252f2ac4c2ba16096aae00baf63 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/A/5bb00252f2ac4c2ba16096aae00baf63 2024-11-10T22:40:26,168 DEBUG [StoreCloser-TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/A/d326aec03d63402391f443d83fcdd388 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/A/d326aec03d63402391f443d83fcdd388 2024-11-10T22:40:26,169 DEBUG [StoreCloser-TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/A/88596cd3a25247289509b1f2dffd22e2 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/A/88596cd3a25247289509b1f2dffd22e2 2024-11-10T22:40:26,170 DEBUG [StoreCloser-TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/A/5433176809c04e3a80aab221358da7ff to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/A/5433176809c04e3a80aab221358da7ff 2024-11-10T22:40:26,171 DEBUG [StoreCloser-TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/A/8f55b2bc7b6143bd8befb0e53f4f9b67 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/A/8f55b2bc7b6143bd8befb0e53f4f9b67 2024-11-10T22:40:26,171 DEBUG [StoreCloser-TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/A/8bb0ba8e0e74432fa92d64f0dbf58fca to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/A/8bb0ba8e0e74432fa92d64f0dbf58fca 2024-11-10T22:40:26,172 DEBUG [StoreCloser-TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/A/22c4ee09e6ac4d4ead24e42e776dcb61 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/A/22c4ee09e6ac4d4ead24e42e776dcb61 2024-11-10T22:40:26,173 DEBUG [StoreCloser-TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/A/b61fd364dc6f402f95b8255cf5523d0e to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/A/b61fd364dc6f402f95b8255cf5523d0e 2024-11-10T22:40:26,174 DEBUG [StoreCloser-TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/A/33c945fa5df44e238d7955533050f24f to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/A/33c945fa5df44e238d7955533050f24f 2024-11-10T22:40:26,175 DEBUG [StoreCloser-TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/A/5fcb329703ad42c0824d3138bf46c17b to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/A/5fcb329703ad42c0824d3138bf46c17b 2024-11-10T22:40:26,176 DEBUG [StoreCloser-TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/A/d0b2da00267449a0a239c334df436bd8 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/A/d0b2da00267449a0a239c334df436bd8 2024-11-10T22:40:26,177 DEBUG [StoreCloser-TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/A/fa5df9fffc994140b6c7023cc37d1331 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/A/fa5df9fffc994140b6c7023cc37d1331 2024-11-10T22:40:26,178 DEBUG [StoreCloser-TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/B/b2c5fd95eaf0403290c6cc9cbb96e8e1, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/B/b1e43329d5874f0ea1d8bfcd751c884e, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/B/91ba298ae7734a59b9802df3fe5088ee, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/B/0f14e05a94c149d897979563f931d484, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/B/8fc13086e9204f7f81d0d2355e283e60, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/B/0204fa2d7c374c04a71d60ea8e01011c, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/B/e45c5d066a5440b28dd7e7b6396047f3, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/B/860d981fe0e74056be6e75fb76e51402, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/B/801b04ffce4146908318d927de243dd3, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/B/668d43b6300e4e7d8334e5592add588b, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/B/9a26618c255c41909078d0a75023b483, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/B/89ade9a388cd41358c4a6fa5653b3bfb, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/B/02b3d13f9cbf4709be56b125fe6f6819, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/B/a8d020254cd2462fa7db7fb4e5403ab0, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/B/ce707a9fc47f4be0864d721e6fbf8a76, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/B/542b81ece6024aa3bc22151d6958c918, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/B/27c2e664e93643909b90f14623f710af, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/B/fd5b6ac2fa084096953ce9b0a4a87e97, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/B/36e8d6d6110b4aa4b64afca826d74230, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/B/9f3a64fb4b9e471ca361a37764b6f0ae, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/B/cc842154b05f48339fea58268b2072f3, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/B/1d9e8d83a44d407a82e27c62f3222b16, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/B/b5d610445f4244b5a989cf8aa882b614, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/B/ed1fa467480743909b8b7672389c25f3, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/B/cea867e6e9184561940cb89e3fd30930] to archive 2024-11-10T22:40:26,178 DEBUG [StoreCloser-TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T22:40:26,179 DEBUG [StoreCloser-TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/B/b2c5fd95eaf0403290c6cc9cbb96e8e1 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/B/b2c5fd95eaf0403290c6cc9cbb96e8e1 2024-11-10T22:40:26,180 DEBUG [StoreCloser-TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/B/b1e43329d5874f0ea1d8bfcd751c884e to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/B/b1e43329d5874f0ea1d8bfcd751c884e 2024-11-10T22:40:26,181 DEBUG [StoreCloser-TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/B/91ba298ae7734a59b9802df3fe5088ee to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/B/91ba298ae7734a59b9802df3fe5088ee 2024-11-10T22:40:26,182 DEBUG [StoreCloser-TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/B/0f14e05a94c149d897979563f931d484 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/B/0f14e05a94c149d897979563f931d484 2024-11-10T22:40:26,183 DEBUG [StoreCloser-TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/B/8fc13086e9204f7f81d0d2355e283e60 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/B/8fc13086e9204f7f81d0d2355e283e60 2024-11-10T22:40:26,183 DEBUG [StoreCloser-TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/B/0204fa2d7c374c04a71d60ea8e01011c to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/B/0204fa2d7c374c04a71d60ea8e01011c 2024-11-10T22:40:26,184 DEBUG [StoreCloser-TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/B/e45c5d066a5440b28dd7e7b6396047f3 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/B/e45c5d066a5440b28dd7e7b6396047f3 2024-11-10T22:40:26,185 DEBUG [StoreCloser-TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/B/860d981fe0e74056be6e75fb76e51402 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/B/860d981fe0e74056be6e75fb76e51402 2024-11-10T22:40:26,185 DEBUG [StoreCloser-TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/B/801b04ffce4146908318d927de243dd3 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/B/801b04ffce4146908318d927de243dd3 2024-11-10T22:40:26,186 DEBUG [StoreCloser-TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/B/668d43b6300e4e7d8334e5592add588b to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/B/668d43b6300e4e7d8334e5592add588b 2024-11-10T22:40:26,187 DEBUG [StoreCloser-TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/B/9a26618c255c41909078d0a75023b483 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/B/9a26618c255c41909078d0a75023b483 2024-11-10T22:40:26,188 DEBUG [StoreCloser-TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/B/89ade9a388cd41358c4a6fa5653b3bfb to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/B/89ade9a388cd41358c4a6fa5653b3bfb 2024-11-10T22:40:26,189 DEBUG [StoreCloser-TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/B/02b3d13f9cbf4709be56b125fe6f6819 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/B/02b3d13f9cbf4709be56b125fe6f6819 2024-11-10T22:40:26,190 DEBUG [StoreCloser-TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/B/a8d020254cd2462fa7db7fb4e5403ab0 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/B/a8d020254cd2462fa7db7fb4e5403ab0 2024-11-10T22:40:26,190 DEBUG [StoreCloser-TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/B/ce707a9fc47f4be0864d721e6fbf8a76 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/B/ce707a9fc47f4be0864d721e6fbf8a76 2024-11-10T22:40:26,191 DEBUG [StoreCloser-TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/B/542b81ece6024aa3bc22151d6958c918 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/B/542b81ece6024aa3bc22151d6958c918 2024-11-10T22:40:26,192 DEBUG [StoreCloser-TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/B/27c2e664e93643909b90f14623f710af to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/B/27c2e664e93643909b90f14623f710af 2024-11-10T22:40:26,193 DEBUG [StoreCloser-TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/B/fd5b6ac2fa084096953ce9b0a4a87e97 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/B/fd5b6ac2fa084096953ce9b0a4a87e97 2024-11-10T22:40:26,194 DEBUG [StoreCloser-TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/B/36e8d6d6110b4aa4b64afca826d74230 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/B/36e8d6d6110b4aa4b64afca826d74230 2024-11-10T22:40:26,195 DEBUG [StoreCloser-TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/B/9f3a64fb4b9e471ca361a37764b6f0ae to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/B/9f3a64fb4b9e471ca361a37764b6f0ae 2024-11-10T22:40:26,195 DEBUG [StoreCloser-TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/B/cc842154b05f48339fea58268b2072f3 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/B/cc842154b05f48339fea58268b2072f3 2024-11-10T22:40:26,196 DEBUG [StoreCloser-TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/B/1d9e8d83a44d407a82e27c62f3222b16 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/B/1d9e8d83a44d407a82e27c62f3222b16 2024-11-10T22:40:26,197 DEBUG [StoreCloser-TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/B/b5d610445f4244b5a989cf8aa882b614 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/B/b5d610445f4244b5a989cf8aa882b614 2024-11-10T22:40:26,198 DEBUG [StoreCloser-TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/B/ed1fa467480743909b8b7672389c25f3 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/B/ed1fa467480743909b8b7672389c25f3 2024-11-10T22:40:26,198 DEBUG [StoreCloser-TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/B/cea867e6e9184561940cb89e3fd30930 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/B/cea867e6e9184561940cb89e3fd30930 2024-11-10T22:40:26,199 DEBUG [StoreCloser-TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/C/449774d5398a42fe9cea117b9dba4146, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/C/fa9dfce9206946cd954a5aa298a1bc0c, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/C/fd68d70a7314421f8bfdf96aa0828b06, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/C/bacedf3bb2194116a50d978460f44dfd, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/C/747e3bb9244f4960a2dfb96d21e0a3b2, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/C/ca57369bdef3452ebbb4762bca3eec0b, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/C/097462359a8943d88ed4fbd1ce86e5cb, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/C/0dc098c699404e9bae5409f495bd925a, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/C/f0fc695ba3324d359a84fbb9d6e46530, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/C/d8431a76a5284f6bbf4f79a13587e982, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/C/8e70d49d390c417a8be3be74f37e7650, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/C/ad52d63d456d414aa7106075af23fa18, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/C/573662096e5648368dece26a3d6ee404, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/C/cc0aa2179752406fa3ca17cc0ac9f39a, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/C/f73616b4ec2649f6b0cafc425657b3a8, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/C/e789fd6415ea4a87abfc52b8ef1c781e, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/C/4e9e2017ac9947448b8e2ceddd2d90cc, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/C/ef53a3a65458426d9865d5be2aa28f20, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/C/fc22149d10a1474c9135a1521d87d6df, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/C/4ed331405a8f489f86167e6c107ab8b7, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/C/05f485b15db94b55a2d475d50bacd1ff, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/C/3b1343f578d24eacbfc6af72614bfd75, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/C/44fad965582845b28d4e779eaf31f10a, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/C/af3d41ce544f4674bccf87c2d844fb2b] to archive 2024-11-10T22:40:26,200 DEBUG [StoreCloser-TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T22:40:26,201 DEBUG [StoreCloser-TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/C/449774d5398a42fe9cea117b9dba4146 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/C/449774d5398a42fe9cea117b9dba4146 2024-11-10T22:40:26,202 DEBUG [StoreCloser-TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/C/fa9dfce9206946cd954a5aa298a1bc0c to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/C/fa9dfce9206946cd954a5aa298a1bc0c 2024-11-10T22:40:26,203 DEBUG [StoreCloser-TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/C/fd68d70a7314421f8bfdf96aa0828b06 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/C/fd68d70a7314421f8bfdf96aa0828b06 2024-11-10T22:40:26,203 DEBUG [StoreCloser-TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/C/bacedf3bb2194116a50d978460f44dfd to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/C/bacedf3bb2194116a50d978460f44dfd 2024-11-10T22:40:26,204 DEBUG [StoreCloser-TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/C/747e3bb9244f4960a2dfb96d21e0a3b2 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/C/747e3bb9244f4960a2dfb96d21e0a3b2 2024-11-10T22:40:26,205 DEBUG [StoreCloser-TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/C/ca57369bdef3452ebbb4762bca3eec0b to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/C/ca57369bdef3452ebbb4762bca3eec0b 2024-11-10T22:40:26,205 DEBUG [StoreCloser-TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/C/097462359a8943d88ed4fbd1ce86e5cb to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/C/097462359a8943d88ed4fbd1ce86e5cb 2024-11-10T22:40:26,206 DEBUG [StoreCloser-TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/C/0dc098c699404e9bae5409f495bd925a to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/C/0dc098c699404e9bae5409f495bd925a 2024-11-10T22:40:26,207 DEBUG [StoreCloser-TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/C/f0fc695ba3324d359a84fbb9d6e46530 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/C/f0fc695ba3324d359a84fbb9d6e46530 2024-11-10T22:40:26,208 DEBUG [StoreCloser-TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/C/d8431a76a5284f6bbf4f79a13587e982 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/C/d8431a76a5284f6bbf4f79a13587e982 2024-11-10T22:40:26,208 DEBUG [StoreCloser-TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/C/8e70d49d390c417a8be3be74f37e7650 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/C/8e70d49d390c417a8be3be74f37e7650 2024-11-10T22:40:26,209 DEBUG [StoreCloser-TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/C/ad52d63d456d414aa7106075af23fa18 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/C/ad52d63d456d414aa7106075af23fa18 2024-11-10T22:40:26,210 DEBUG [StoreCloser-TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/C/573662096e5648368dece26a3d6ee404 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/C/573662096e5648368dece26a3d6ee404 2024-11-10T22:40:26,210 DEBUG [StoreCloser-TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/C/cc0aa2179752406fa3ca17cc0ac9f39a to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/C/cc0aa2179752406fa3ca17cc0ac9f39a 2024-11-10T22:40:26,211 DEBUG [StoreCloser-TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/C/f73616b4ec2649f6b0cafc425657b3a8 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/C/f73616b4ec2649f6b0cafc425657b3a8 2024-11-10T22:40:26,212 DEBUG [StoreCloser-TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/C/e789fd6415ea4a87abfc52b8ef1c781e to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/C/e789fd6415ea4a87abfc52b8ef1c781e 2024-11-10T22:40:26,213 DEBUG [StoreCloser-TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/C/4e9e2017ac9947448b8e2ceddd2d90cc to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/C/4e9e2017ac9947448b8e2ceddd2d90cc 2024-11-10T22:40:26,214 DEBUG [StoreCloser-TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/C/ef53a3a65458426d9865d5be2aa28f20 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/C/ef53a3a65458426d9865d5be2aa28f20 2024-11-10T22:40:26,214 DEBUG [StoreCloser-TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/C/fc22149d10a1474c9135a1521d87d6df to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/C/fc22149d10a1474c9135a1521d87d6df 2024-11-10T22:40:26,215 DEBUG [StoreCloser-TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/C/4ed331405a8f489f86167e6c107ab8b7 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/C/4ed331405a8f489f86167e6c107ab8b7 2024-11-10T22:40:26,216 DEBUG [StoreCloser-TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/C/05f485b15db94b55a2d475d50bacd1ff to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/C/05f485b15db94b55a2d475d50bacd1ff 2024-11-10T22:40:26,217 DEBUG [StoreCloser-TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/C/3b1343f578d24eacbfc6af72614bfd75 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/C/3b1343f578d24eacbfc6af72614bfd75 2024-11-10T22:40:26,217 DEBUG [StoreCloser-TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/C/44fad965582845b28d4e779eaf31f10a to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/C/44fad965582845b28d4e779eaf31f10a 2024-11-10T22:40:26,218 DEBUG [StoreCloser-TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/C/af3d41ce544f4674bccf87c2d844fb2b to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/C/af3d41ce544f4674bccf87c2d844fb2b 2024-11-10T22:40:26,222 DEBUG [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/recovered.edits/362.seqid, newMaxSeqId=362, maxSeqId=4 2024-11-10T22:40:26,223 INFO [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. 2024-11-10T22:40:26,223 DEBUG [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] regionserver.HRegion(1676): Region close journal for 74244978b603a65b5f2074c624cb7957: Waiting for close lock at 1731278424502Running coprocessor pre-close hooks at 1731278424502Disabling compacts and flushes for region at 1731278424502Disabling writes for close at 1731278424502Obtaining lock to block concurrent updates at 1731278424502Preparing flush snapshotting stores in 74244978b603a65b5f2074c624cb7957 at 1731278424502Finished memstore snapshotting TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957., syncing WAL and waiting on mvcc, flushsize=dataSize=13740, getHeapSize=36720, getOffHeapSize=0, getCellsCount=300 at 1731278424503 (+1 ms)Flushing stores of TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957. at 1731278424503Flushing 74244978b603a65b5f2074c624cb7957/A: creating writer at 1731278424504 (+1 ms)Flushing mob file 74244978b603a65b5f2074c624cb7957/A: appending metadata at 1731278424507 (+3 ms)Flushing mob file 74244978b603a65b5f2074c624cb7957/A: closing flushed file at 1731278424507Flushing 74244978b603a65b5f2074c624cb7957/A: appending metadata at 1731278424915 (+408 ms)Flushing 74244978b603a65b5f2074c624cb7957/A: closing flushed file at 1731278424915Flushing 74244978b603a65b5f2074c624cb7957/B: creating writer at 1731278425322 (+407 ms)Flushing 74244978b603a65b5f2074c624cb7957/B: appending metadata at 1731278425324 (+2 ms)Flushing 74244978b603a65b5f2074c624cb7957/B: closing flushed file at 1731278425324Flushing 74244978b603a65b5f2074c624cb7957/C: creating writer at 1731278425735 (+411 ms)Flushing 74244978b603a65b5f2074c624cb7957/C: appending metadata at 1731278425737 (+2 ms)Flushing 74244978b603a65b5f2074c624cb7957/C: closing flushed file at 1731278425737Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@175f042e: reopening flushed file at 1731278426144 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5758d902: reopening flushed file at 1731278426147 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@73cbdd5e: reopening flushed file at 1731278426151 (+4 ms)Finished flush of dataSize ~13.42 KB/13740, heapSize ~35.86 KB/36720, currentSize=0 B/0 for 74244978b603a65b5f2074c624cb7957 in 1653ms, sequenceid=359, compaction requested=false at 1731278426155 (+4 ms)Writing region close event to WAL at 1731278426219 (+64 ms)Running coprocessor post-close hooks at 1731278426223 (+4 ms)Closed at 1731278426223 2024-11-10T22:40:26,225 INFO [RS_CLOSE_REGION-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] handler.UnassignRegionHandler(157): Closed 74244978b603a65b5f2074c624cb7957 2024-11-10T22:40:26,225 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=178 updating hbase:meta row=74244978b603a65b5f2074c624cb7957, regionState=CLOSED 2024-11-10T22:40:26,227 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=179, ppid=178, state=RUNNABLE, hasLock=false; CloseRegionProcedure 74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 because future has completed 2024-11-10T22:40:26,229 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=179, resume processing ppid=178 2024-11-10T22:40:26,229 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=179, ppid=178, state=SUCCESS, hasLock=false; CloseRegionProcedure 74244978b603a65b5f2074c624cb7957, server=849d0d71310f,43753,1731278250663 in 1.8780 sec 2024-11-10T22:40:26,231 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=178, resume processing ppid=177 2024-11-10T22:40:26,231 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=178, ppid=177, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=74244978b603a65b5f2074c624cb7957, UNASSIGN in 1.8840 sec 2024-11-10T22:40:26,232 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=177, resume processing ppid=176 2024-11-10T22:40:26,232 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=177, ppid=176, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8870 sec 2024-11-10T22:40:26,234 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731278426233"}]},"ts":"1731278426233"} 2024-11-10T22:40:26,235 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-10T22:40:26,235 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-10T22:40:26,236 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=176, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8980 sec 2024-11-10T22:40:26,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=176 2024-11-10T22:40:26,483 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-10T22:40:26,484 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-10T22:40:26,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] procedure2.ProcedureExecutor(1139): Stored pid=180, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T22:40:26,485 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=180, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T22:40:26,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=180 2024-11-10T22:40:26,486 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=180, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T22:40:26,489 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957 2024-11-10T22:40:26,491 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/A, FileablePath, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/B, FileablePath, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/C, FileablePath, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/recovered.edits] 2024-11-10T22:40:26,492 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/A/0a8aa1ea6ab64ae68dc0eb843c2d9ed0 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/A/0a8aa1ea6ab64ae68dc0eb843c2d9ed0 2024-11-10T22:40:26,493 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/A/cb1574506b394cf7915477e70a330c35 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/A/cb1574506b394cf7915477e70a330c35 2024-11-10T22:40:26,495 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/B/654c8ae0be1f41e198b766e0060ce254 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/B/654c8ae0be1f41e198b766e0060ce254 2024-11-10T22:40:26,495 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/B/a2befa0e8a4b48bf86422a3c1a63333d to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/B/a2befa0e8a4b48bf86422a3c1a63333d 2024-11-10T22:40:26,497 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/C/1712c96782754560af0803805e05ff11 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/C/1712c96782754560af0803805e05ff11 2024-11-10T22:40:26,497 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/C/1cde88902b1242c7bc104bd3eb41b3a7 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/C/1cde88902b1242c7bc104bd3eb41b3a7 2024-11-10T22:40:26,499 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/recovered.edits/362.seqid to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957/recovered.edits/362.seqid 2024-11-10T22:40:26,500 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/default/TestAcidGuarantees/74244978b603a65b5f2074c624cb7957 2024-11-10T22:40:26,500 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-10T22:40:26,500 DEBUG [PEWorker-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-10T22:40:26,501 DEBUG [PEWorker-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-10T22:40:26,503 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111005212343fb6441e0b2fd60ee273486cb_74244978b603a65b5f2074c624cb7957 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111005212343fb6441e0b2fd60ee273486cb_74244978b603a65b5f2074c624cb7957 2024-11-10T22:40:26,504 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411100e8d2180d8f5473899114e5e80a51652_74244978b603a65b5f2074c624cb7957 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411100e8d2180d8f5473899114e5e80a51652_74244978b603a65b5f2074c624cb7957 2024-11-10T22:40:26,505 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411102693968e03bf44b0aff72f0537a69825_74244978b603a65b5f2074c624cb7957 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411102693968e03bf44b0aff72f0537a69825_74244978b603a65b5f2074c624cb7957 2024-11-10T22:40:26,506 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411102d3c0f4d1ae94508a31dca5da3a70663_74244978b603a65b5f2074c624cb7957 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411102d3c0f4d1ae94508a31dca5da3a70663_74244978b603a65b5f2074c624cb7957 2024-11-10T22:40:26,507 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110369aa87bd4e9446e86fdf05223ddf861_74244978b603a65b5f2074c624cb7957 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110369aa87bd4e9446e86fdf05223ddf861_74244978b603a65b5f2074c624cb7957 2024-11-10T22:40:26,508 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110486b44ec56a7403899edefe20ec943b1_74244978b603a65b5f2074c624cb7957 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110486b44ec56a7403899edefe20ec943b1_74244978b603a65b5f2074c624cb7957 2024-11-10T22:40:26,509 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411104bdf589343104db9b0ef37048cc75616_74244978b603a65b5f2074c624cb7957 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411104bdf589343104db9b0ef37048cc75616_74244978b603a65b5f2074c624cb7957 2024-11-10T22:40:26,510 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411105663d9d51f694397a672bbdcfcec5328_74244978b603a65b5f2074c624cb7957 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411105663d9d51f694397a672bbdcfcec5328_74244978b603a65b5f2074c624cb7957 2024-11-10T22:40:26,511 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411107352bf8ea63e4344a04b583d681600c7_74244978b603a65b5f2074c624cb7957 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411107352bf8ea63e4344a04b583d681600c7_74244978b603a65b5f2074c624cb7957 2024-11-10T22:40:26,512 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411107ba04c79dcb046a48ddcda55f64a076e_74244978b603a65b5f2074c624cb7957 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411107ba04c79dcb046a48ddcda55f64a076e_74244978b603a65b5f2074c624cb7957 2024-11-10T22:40:26,513 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411108f323fede0b54025b08ed3c953d3a6ba_74244978b603a65b5f2074c624cb7957 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411108f323fede0b54025b08ed3c953d3a6ba_74244978b603a65b5f2074c624cb7957 2024-11-10T22:40:26,514 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411109bd027f30c03421e8e19a33d3f4e4245_74244978b603a65b5f2074c624cb7957 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411109bd027f30c03421e8e19a33d3f4e4245_74244978b603a65b5f2074c624cb7957 2024-11-10T22:40:26,515 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110a20f1b6e64484bd386b1f81634379bd3_74244978b603a65b5f2074c624cb7957 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110a20f1b6e64484bd386b1f81634379bd3_74244978b603a65b5f2074c624cb7957 2024-11-10T22:40:26,516 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110a82efc91d2cf446abe8dd51198e85cb1_74244978b603a65b5f2074c624cb7957 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110a82efc91d2cf446abe8dd51198e85cb1_74244978b603a65b5f2074c624cb7957 2024-11-10T22:40:26,517 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110dcd91692e0a047288de3cee2f39e81ce_74244978b603a65b5f2074c624cb7957 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110dcd91692e0a047288de3cee2f39e81ce_74244978b603a65b5f2074c624cb7957 2024-11-10T22:40:26,517 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110e435529fa79d480ead34ba1b8e556065_74244978b603a65b5f2074c624cb7957 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110e435529fa79d480ead34ba1b8e556065_74244978b603a65b5f2074c624cb7957 2024-11-10T22:40:26,518 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110e94e578ae286451bba00186959f26208_74244978b603a65b5f2074c624cb7957 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110e94e578ae286451bba00186959f26208_74244978b603a65b5f2074c624cb7957 2024-11-10T22:40:26,519 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110ee90dd6d4ad74b9bbb7e6481725e5cd7_74244978b603a65b5f2074c624cb7957 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110ee90dd6d4ad74b9bbb7e6481725e5cd7_74244978b603a65b5f2074c624cb7957 2024-11-10T22:40:26,524 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110f14d130e4cc64f7c9c217f3b894ffca2_74244978b603a65b5f2074c624cb7957 to hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110f14d130e4cc64f7c9c217f3b894ffca2_74244978b603a65b5f2074c624cb7957 2024-11-10T22:40:26,525 DEBUG [PEWorker-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-10T22:40:26,527 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=180, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T22:40:26,529 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-10T22:40:26,531 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-10T22:40:26,532 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=180, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T22:40:26,532 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-10T22:40:26,533 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731278426532"}]},"ts":"9223372036854775807"} 2024-11-10T22:40:26,534 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-10T22:40:26,534 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 74244978b603a65b5f2074c624cb7957, NAME => 'TestAcidGuarantees,,1731278397582.74244978b603a65b5f2074c624cb7957.', STARTKEY => '', ENDKEY => ''}] 2024-11-10T22:40:26,534 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-10T22:40:26,535 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731278426534"}]},"ts":"9223372036854775807"} 2024-11-10T22:40:26,536 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-10T22:40:26,537 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=180, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T22:40:26,538 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=180, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 53 msec 2024-11-10T22:40:26,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=180 2024-11-10T22:40:26,593 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-10T22:40:26,593 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-10T22:40:26,604 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobGetAtomicity Thread=243 (was 240) - Thread LEAK? -, OpenFileDescriptor=457 (was 449) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=280 (was 275) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=7840 (was 6927) - AvailableMemoryMB LEAK? - 2024-11-10T22:40:26,604 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-10T22:40:26,604 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-10T22:40:26,604 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T22:40:26,604 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:40:26,604 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:40:26,604 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T22:40:26,604 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-10T22:40:26,604 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=197894091, stopped=false 2024-11-10T22:40:26,605 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=849d0d71310f,43351,1731278249979 2024-11-10T22:40:26,607 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43351-0x101945512150000, quorum=127.0.0.1:53159, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-10T22:40:26,607 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43351-0x101945512150000, quorum=127.0.0.1:53159, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T22:40:26,607 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43753-0x101945512150001, quorum=127.0.0.1:53159, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-10T22:40:26,607 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43753-0x101945512150001, quorum=127.0.0.1:53159, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T22:40:26,607 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-10T22:40:26,608 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-10T22:40:26,608 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:43753-0x101945512150001, quorum=127.0.0.1:53159, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-10T22:40:26,608 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T22:40:26,608 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:40:26,608 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:43351-0x101945512150000, quorum=127.0.0.1:53159, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-10T22:40:26,609 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '849d0d71310f,43753,1731278250663' ***** 2024-11-10T22:40:26,609 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-10T22:40:26,609 INFO [RS:0;849d0d71310f:43753 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-10T22:40:26,609 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-10T22:40:26,610 INFO [RS:0;849d0d71310f:43753 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-10T22:40:26,610 INFO [RS:0;849d0d71310f:43753 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-10T22:40:26,610 INFO [RS:0;849d0d71310f:43753 {}] regionserver.HRegionServer(959): stopping server 849d0d71310f,43753,1731278250663 2024-11-10T22:40:26,610 INFO [RS:0;849d0d71310f:43753 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-10T22:40:26,610 INFO [RS:0;849d0d71310f:43753 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;849d0d71310f:43753. 2024-11-10T22:40:26,610 DEBUG [RS:0;849d0d71310f:43753 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T22:40:26,610 DEBUG [RS:0;849d0d71310f:43753 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:40:26,610 INFO [RS:0;849d0d71310f:43753 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-10T22:40:26,610 INFO [RS:0;849d0d71310f:43753 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-10T22:40:26,610 INFO [RS:0;849d0d71310f:43753 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-10T22:40:26,610 INFO [RS:0;849d0d71310f:43753 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-10T22:40:26,611 INFO [RS:0;849d0d71310f:43753 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-10T22:40:26,611 DEBUG [RS:0;849d0d71310f:43753 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-10T22:40:26,612 DEBUG [RS:0;849d0d71310f:43753 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-10T22:40:26,612 DEBUG [RS_CLOSE_META-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-10T22:40:26,612 INFO [RS_CLOSE_META-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-10T22:40:26,612 DEBUG [RS_CLOSE_META-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-10T22:40:26,612 DEBUG [RS_CLOSE_META-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-10T22:40:26,612 DEBUG [RS_CLOSE_META-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-10T22:40:26,612 INFO [RS_CLOSE_META-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-11-10T22:40:26,615 INFO [regionserver/849d0d71310f:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-10T22:40:26,634 DEBUG [RS_CLOSE_META-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/hbase/meta/1588230740/.tmp/info/6d748a37ab93441a9696f707f5229c94 is 95, key is TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339./info:/1731278286611/DeleteFamily/seqid=0 2024-11-10T22:40:26,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742457_1633 (size=6514) 2024-11-10T22:40:26,812 DEBUG [RS:0;849d0d71310f:43753 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-10T22:40:27,012 DEBUG [RS:0;849d0d71310f:43753 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-10T22:40:27,039 INFO [RS_CLOSE_META-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/hbase/meta/1588230740/.tmp/info/6d748a37ab93441a9696f707f5229c94 2024-11-10T22:40:27,065 DEBUG [RS_CLOSE_META-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/hbase/meta/1588230740/.tmp/ns/df604ebfcf4f4fc581920fb82bdf865b is 93, key is TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339./ns:/1731278286571/DeleteFamily/seqid=0 2024-11-10T22:40:27,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742458_1634 (size=5906) 2024-11-10T22:40:27,077 INFO [RS_CLOSE_META-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/hbase/meta/1588230740/.tmp/ns/df604ebfcf4f4fc581920fb82bdf865b 2024-11-10T22:40:27,102 DEBUG [RS_CLOSE_META-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/hbase/meta/1588230740/.tmp/rep_barrier/9ee9856795f8470a9676db7779371583 is 102, key is TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339./rep_barrier:/1731278286571/DeleteFamily/seqid=0 2024-11-10T22:40:27,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742459_1635 (size=6025) 2024-11-10T22:40:27,115 INFO [RS_CLOSE_META-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/hbase/meta/1588230740/.tmp/rep_barrier/9ee9856795f8470a9676db7779371583 2024-11-10T22:40:27,134 DEBUG [RS_CLOSE_META-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/hbase/meta/1588230740/.tmp/table/52e80e932eec452fbacf4c2cea5901db is 96, key is TestAcidGuarantees,,1731278254183.dbbcde27e8d425fcc5932f5841a8c339./table:/1731278286571/DeleteFamily/seqid=0 2024-11-10T22:40:27,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742460_1636 (size=5936) 2024-11-10T22:40:27,164 INFO [regionserver/849d0d71310f:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-10T22:40:27,165 INFO [regionserver/849d0d71310f:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-10T22:40:27,212 DEBUG [RS:0;849d0d71310f:43753 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-10T22:40:27,412 DEBUG [RS:0;849d0d71310f:43753 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-10T22:40:27,538 INFO [RS_CLOSE_META-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/hbase/meta/1588230740/.tmp/table/52e80e932eec452fbacf4c2cea5901db 2024-11-10T22:40:27,542 DEBUG [RS_CLOSE_META-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/hbase/meta/1588230740/.tmp/info/6d748a37ab93441a9696f707f5229c94 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/hbase/meta/1588230740/info/6d748a37ab93441a9696f707f5229c94 2024-11-10T22:40:27,545 INFO [RS_CLOSE_META-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/hbase/meta/1588230740/info/6d748a37ab93441a9696f707f5229c94, entries=12, sequenceid=90, filesize=6.4 K 2024-11-10T22:40:27,546 DEBUG [RS_CLOSE_META-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/hbase/meta/1588230740/.tmp/ns/df604ebfcf4f4fc581920fb82bdf865b as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/hbase/meta/1588230740/ns/df604ebfcf4f4fc581920fb82bdf865b 2024-11-10T22:40:27,549 INFO [RS_CLOSE_META-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/hbase/meta/1588230740/ns/df604ebfcf4f4fc581920fb82bdf865b, entries=8, sequenceid=90, filesize=5.8 K 2024-11-10T22:40:27,550 DEBUG [RS_CLOSE_META-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/hbase/meta/1588230740/.tmp/rep_barrier/9ee9856795f8470a9676db7779371583 as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/hbase/meta/1588230740/rep_barrier/9ee9856795f8470a9676db7779371583 2024-11-10T22:40:27,553 INFO [RS_CLOSE_META-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/hbase/meta/1588230740/rep_barrier/9ee9856795f8470a9676db7779371583, entries=6, sequenceid=90, filesize=5.9 K 2024-11-10T22:40:27,553 DEBUG [RS_CLOSE_META-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/hbase/meta/1588230740/.tmp/table/52e80e932eec452fbacf4c2cea5901db as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/hbase/meta/1588230740/table/52e80e932eec452fbacf4c2cea5901db 2024-11-10T22:40:27,556 INFO [RS_CLOSE_META-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/hbase/meta/1588230740/table/52e80e932eec452fbacf4c2cea5901db, entries=7, sequenceid=90, filesize=5.8 K 2024-11-10T22:40:27,557 INFO [RS_CLOSE_META-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 945ms, sequenceid=90, compaction requested=false 2024-11-10T22:40:27,562 DEBUG [RS_CLOSE_META-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-11-10T22:40:27,562 DEBUG [RS_CLOSE_META-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-10T22:40:27,562 INFO [RS_CLOSE_META-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-10T22:40:27,563 DEBUG [RS_CLOSE_META-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731278426611Running coprocessor pre-close hooks at 1731278426611Disabling compacts and flushes for region at 1731278426611Disabling writes for close at 1731278426612 (+1 ms)Obtaining lock to block concurrent updates at 1731278426612Preparing flush snapshotting stores in 1588230740 at 1731278426612Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1731278426612Flushing stores of hbase:meta,,1.1588230740 at 1731278426613 (+1 ms)Flushing 1588230740/info: creating writer at 1731278426613Flushing 1588230740/info: appending metadata at 1731278426634 (+21 ms)Flushing 1588230740/info: closing flushed file at 1731278426634Flushing 1588230740/ns: creating writer at 1731278427046 (+412 ms)Flushing 1588230740/ns: appending metadata at 1731278427065 (+19 ms)Flushing 1588230740/ns: closing flushed file at 1731278427065Flushing 1588230740/rep_barrier: creating writer at 1731278427081 (+16 ms)Flushing 1588230740/rep_barrier: appending metadata at 1731278427102 (+21 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1731278427102Flushing 1588230740/table: creating writer at 1731278427118 (+16 ms)Flushing 1588230740/table: appending metadata at 1731278427133 (+15 ms)Flushing 1588230740/table: closing flushed file at 1731278427133Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@13a67dbd: reopening flushed file at 1731278427542 (+409 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2309b77f: reopening flushed file at 1731278427546 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@525d7aa7: reopening flushed file at 1731278427549 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@66dd9a18: reopening flushed file at 1731278427553 (+4 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 945ms, sequenceid=90, compaction requested=false at 1731278427557 (+4 ms)Writing region close event to WAL at 1731278427559 (+2 ms)Running coprocessor post-close hooks at 1731278427562 (+3 ms)Closed at 1731278427562 2024-11-10T22:40:27,563 DEBUG [RS_CLOSE_META-regionserver/849d0d71310f:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-10T22:40:27,613 INFO [RS:0;849d0d71310f:43753 {}] regionserver.HRegionServer(976): stopping server 849d0d71310f,43753,1731278250663; all regions closed. 2024-11-10T22:40:27,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073741834_1010 (size=25344) 2024-11-10T22:40:27,621 DEBUG [RS:0;849d0d71310f:43753 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/oldWALs 2024-11-10T22:40:27,621 INFO [RS:0;849d0d71310f:43753 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 849d0d71310f%2C43753%2C1731278250663.meta:.meta(num 1731278253724) 2024-11-10T22:40:27,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073741833_1009 (size=14207247) 2024-11-10T22:40:27,626 DEBUG [RS:0;849d0d71310f:43753 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/oldWALs 2024-11-10T22:40:27,626 INFO [RS:0;849d0d71310f:43753 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 849d0d71310f%2C43753%2C1731278250663:(num 1731278253238) 2024-11-10T22:40:27,626 DEBUG [RS:0;849d0d71310f:43753 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T22:40:27,626 INFO [RS:0;849d0d71310f:43753 {}] regionserver.LeaseManager(133): Closed leases 2024-11-10T22:40:27,626 INFO [RS:0;849d0d71310f:43753 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-10T22:40:27,627 INFO [RS:0;849d0d71310f:43753 {}] hbase.ChoreService(370): Chore service for: regionserver/849d0d71310f:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-10T22:40:27,627 INFO [RS:0;849d0d71310f:43753 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-10T22:40:27,627 INFO [regionserver/849d0d71310f:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-10T22:40:27,628 INFO [RS:0;849d0d71310f:43753 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:43753 2024-11-10T22:40:27,633 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43351-0x101945512150000, quorum=127.0.0.1:53159, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-10T22:40:27,633 INFO [RS:0;849d0d71310f:43753 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-10T22:40:27,633 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43753-0x101945512150001, quorum=127.0.0.1:53159, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/849d0d71310f,43753,1731278250663 2024-11-10T22:40:27,636 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [849d0d71310f,43753,1731278250663] 2024-11-10T22:40:27,638 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/849d0d71310f,43753,1731278250663 already deleted, retry=false 2024-11-10T22:40:27,638 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 849d0d71310f,43753,1731278250663 expired; onlineServers=0 2024-11-10T22:40:27,639 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '849d0d71310f,43351,1731278249979' ***** 2024-11-10T22:40:27,639 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-10T22:40:27,639 INFO [M:0;849d0d71310f:43351 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-10T22:40:27,639 INFO [M:0;849d0d71310f:43351 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-10T22:40:27,639 DEBUG [M:0;849d0d71310f:43351 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-10T22:40:27,639 DEBUG [M:0;849d0d71310f:43351 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-10T22:40:27,639 DEBUG [master/849d0d71310f:0:becomeActiveMaster-HFileCleaner.small.0-1731278252872 {}] cleaner.HFileCleaner(306): Exit Thread[master/849d0d71310f:0:becomeActiveMaster-HFileCleaner.small.0-1731278252872,5,FailOnTimeoutGroup] 2024-11-10T22:40:27,639 DEBUG [master/849d0d71310f:0:becomeActiveMaster-HFileCleaner.large.0-1731278252870 {}] cleaner.HFileCleaner(306): Exit Thread[master/849d0d71310f:0:becomeActiveMaster-HFileCleaner.large.0-1731278252870,5,FailOnTimeoutGroup] 2024-11-10T22:40:27,639 INFO [M:0;849d0d71310f:43351 {}] hbase.ChoreService(370): Chore service for: master/849d0d71310f:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-10T22:40:27,639 INFO [M:0;849d0d71310f:43351 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-10T22:40:27,639 DEBUG [M:0;849d0d71310f:43351 {}] master.HMaster(1795): Stopping service threads 2024-11-10T22:40:27,639 INFO [M:0;849d0d71310f:43351 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-10T22:40:27,640 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-10T22:40:27,640 INFO [M:0;849d0d71310f:43351 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-10T22:40:27,640 ERROR [M:0;849d0d71310f:43351 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[IPC Client (1102635514) connection to localhost/127.0.0.1:44923 from jenkins,5,PEWorkerGroup] Thread[IPC Parameter Sending Thread for localhost/127.0.0.1:44923,5,PEWorkerGroup] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-11-10T22:40:27,641 INFO [M:0;849d0d71310f:43351 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-10T22:40:27,641 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-10T22:40:27,641 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43351-0x101945512150000, quorum=127.0.0.1:53159, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-10T22:40:27,641 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43351-0x101945512150000, quorum=127.0.0.1:53159, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T22:40:27,642 DEBUG [M:0;849d0d71310f:43351 {}] zookeeper.ZKUtil(347): master:43351-0x101945512150000, quorum=127.0.0.1:53159, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-10T22:40:27,642 WARN [M:0;849d0d71310f:43351 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-10T22:40:27,642 INFO [M:0;849d0d71310f:43351 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/.lastflushedseqids 2024-11-10T22:40:27,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742461_1637 (size=72) 2024-11-10T22:40:27,737 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43753-0x101945512150001, quorum=127.0.0.1:53159, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-10T22:40:27,737 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43753-0x101945512150001, quorum=127.0.0.1:53159, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-10T22:40:27,737 INFO [RS:0;849d0d71310f:43753 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-10T22:40:27,737 INFO [RS:0;849d0d71310f:43753 {}] regionserver.HRegionServer(1031): Exiting; stopping=849d0d71310f,43753,1731278250663; zookeeper connection closed. 2024-11-10T22:40:27,738 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@476cadbf {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@476cadbf 2024-11-10T22:40:27,738 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-10T22:40:28,053 INFO [M:0;849d0d71310f:43351 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-10T22:40:28,054 INFO [M:0;849d0d71310f:43351 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-10T22:40:28,054 DEBUG [M:0;849d0d71310f:43351 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-10T22:40:28,054 INFO [M:0;849d0d71310f:43351 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T22:40:28,054 DEBUG [M:0;849d0d71310f:43351 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T22:40:28,054 DEBUG [M:0;849d0d71310f:43351 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-10T22:40:28,054 DEBUG [M:0;849d0d71310f:43351 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T22:40:28,054 INFO [M:0;849d0d71310f:43351 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=767.58 KB heapSize=943.63 KB 2024-11-10T22:40:28,070 DEBUG [M:0;849d0d71310f:43351 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/2b0c77f9514e4c188aed609b2d59fa7c is 82, key is hbase:meta,,1/info:regioninfo/1731278253814/Put/seqid=0 2024-11-10T22:40:28,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742462_1638 (size=5672) 2024-11-10T22:40:28,474 INFO [M:0;849d0d71310f:43351 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2168 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/2b0c77f9514e4c188aed609b2d59fa7c 2024-11-10T22:40:28,503 DEBUG [M:0;849d0d71310f:43351 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/8402593ff7d44930b7ad1a924d1f6e5c is 2276, key is \x00\x00\x00\x00\x00\x00\x00'/proc:d/1731278289794/Put/seqid=0 2024-11-10T22:40:28,508 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742463_1639 (size=45951) 2024-11-10T22:40:28,908 INFO [M:0;849d0d71310f:43351 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=767.03 KB at sequenceid=2168 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/8402593ff7d44930b7ad1a924d1f6e5c 2024-11-10T22:40:28,912 INFO [M:0;849d0d71310f:43351 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 8402593ff7d44930b7ad1a924d1f6e5c 2024-11-10T22:40:28,930 DEBUG [M:0;849d0d71310f:43351 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/011852fa9abf4be7ac0ced4b82dc85cb is 69, key is 849d0d71310f,43753,1731278250663/rs:state/1731278253001/Put/seqid=0 2024-11-10T22:40:28,937 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073742464_1640 (size=5156) 2024-11-10T22:40:29,064 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-10T22:40:29,338 INFO [M:0;849d0d71310f:43351 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2168 (bloomFilter=true), to=hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/011852fa9abf4be7ac0ced4b82dc85cb 2024-11-10T22:40:29,341 DEBUG [M:0;849d0d71310f:43351 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/2b0c77f9514e4c188aed609b2d59fa7c as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/2b0c77f9514e4c188aed609b2d59fa7c 2024-11-10T22:40:29,344 INFO [M:0;849d0d71310f:43351 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/2b0c77f9514e4c188aed609b2d59fa7c, entries=8, sequenceid=2168, filesize=5.5 K 2024-11-10T22:40:29,345 DEBUG [M:0;849d0d71310f:43351 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/8402593ff7d44930b7ad1a924d1f6e5c as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/8402593ff7d44930b7ad1a924d1f6e5c 2024-11-10T22:40:29,348 INFO [M:0;849d0d71310f:43351 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 8402593ff7d44930b7ad1a924d1f6e5c 2024-11-10T22:40:29,348 INFO [M:0;849d0d71310f:43351 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/8402593ff7d44930b7ad1a924d1f6e5c, entries=180, sequenceid=2168, filesize=44.9 K 2024-11-10T22:40:29,349 DEBUG [M:0;849d0d71310f:43351 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/011852fa9abf4be7ac0ced4b82dc85cb as hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/011852fa9abf4be7ac0ced4b82dc85cb 2024-11-10T22:40:29,352 INFO [M:0;849d0d71310f:43351 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44923/user/jenkins/test-data/ff5e4594-df82-1682-5bdd-792692a55f39/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/011852fa9abf4be7ac0ced4b82dc85cb, entries=1, sequenceid=2168, filesize=5.0 K 2024-11-10T22:40:29,353 INFO [M:0;849d0d71310f:43351 {}] regionserver.HRegion(3140): Finished flush of dataSize ~767.58 KB/786005, heapSize ~943.33 KB/965968, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1299ms, sequenceid=2168, compaction requested=false 2024-11-10T22:40:29,354 INFO [M:0;849d0d71310f:43351 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T22:40:29,354 DEBUG [M:0;849d0d71310f:43351 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731278428054Disabling compacts and flushes for region at 1731278428054Disabling writes for close at 1731278428054Obtaining lock to block concurrent updates at 1731278428054Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731278428054Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=786005, getHeapSize=966208, getOffHeapSize=0, getCellsCount=2373 at 1731278428054Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731278428055 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731278428055Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731278428070 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731278428070Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731278428478 (+408 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731278428503 (+25 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731278428503Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731278428912 (+409 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731278428929 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731278428929Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1d866e14: reopening flushed file at 1731278429341 (+412 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1e98f22f: reopening flushed file at 1731278429345 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@64857b5c: reopening flushed file at 1731278429348 (+3 ms)Finished flush of dataSize ~767.58 KB/786005, heapSize ~943.33 KB/965968, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1299ms, sequenceid=2168, compaction requested=false at 1731278429353 (+5 ms)Writing region close event to WAL at 1731278429354 (+1 ms)Closed at 1731278429354 2024-11-10T22:40:29,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46149 is added to blk_1073741830_1006 (size=928164) 2024-11-10T22:40:29,357 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-10T22:40:29,357 INFO [M:0;849d0d71310f:43351 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-10T22:40:29,357 INFO [M:0;849d0d71310f:43351 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:43351 2024-11-10T22:40:29,358 INFO [M:0;849d0d71310f:43351 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-10T22:40:29,460 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43351-0x101945512150000, quorum=127.0.0.1:53159, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-10T22:40:29,460 INFO [M:0;849d0d71310f:43351 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-10T22:40:29,460 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43351-0x101945512150000, quorum=127.0.0.1:53159, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-10T22:40:29,465 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4269bd31{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-10T22:40:29,467 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-10T22:40:29,468 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-10T22:40:29,468 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-10T22:40:29,468 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3340d2ba-2e32-7278-81af-9687863d92cc/hadoop.log.dir/,STOPPED} 2024-11-10T22:40:29,471 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-10T22:40:29,471 WARN [BP-1304540299-172.17.0.2-1731278247130 heartbeating to localhost/127.0.0.1:44923 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-10T22:40:29,471 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-10T22:40:29,471 WARN [BP-1304540299-172.17.0.2-1731278247130 heartbeating to localhost/127.0.0.1:44923 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1304540299-172.17.0.2-1731278247130 (Datanode Uuid 96a73bbd-4590-4882-973d-216f9d2ea6ca) service to localhost/127.0.0.1:44923 2024-11-10T22:40:29,474 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3340d2ba-2e32-7278-81af-9687863d92cc/cluster_f1d987d6-b0d7-db14-6c2b-f9cd24bc1de9/data/data1/current/BP-1304540299-172.17.0.2-1731278247130 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-10T22:40:29,475 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3340d2ba-2e32-7278-81af-9687863d92cc/cluster_f1d987d6-b0d7-db14-6c2b-f9cd24bc1de9/data/data2/current/BP-1304540299-172.17.0.2-1731278247130 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-10T22:40:29,475 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-10T22:40:29,485 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4ded99d3{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-10T22:40:29,485 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-10T22:40:29,486 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-10T22:40:29,486 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-10T22:40:29,486 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3340d2ba-2e32-7278-81af-9687863d92cc/hadoop.log.dir/,STOPPED} 2024-11-10T22:40:29,502 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-10T22:40:29,631 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down